Generated at: 2025-05-14 00:26:50
{"level":"warn","ts":"2025-04-25T06:04:16.458245Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.002736949s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"}
{"level":"warn","ts":"2025-04-25T06:09:16.452199Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.007711416s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"}
{"level":"warn","ts":"2025-04-25T06:14:16.445792Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000352598s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"}
{"level":"warn","ts":"2025-04-25T06:24:17.342252Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000196896s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"}
{"level":"warn","ts":"2025-04-25T06:24:17.444017Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.999925858s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"182"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","2654f0dc-728a-44ab-86c2-2c90e7b69f77","--dag_execution_id","182","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-54j6v/2025/05/13/kubeflow-cluster-report-html-54j6v-system-container-driver-1370600769"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"182"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","2654f0dc-728a-44ab-86c2-2c90e7b69f77","--dag_execution_id","182","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-54j6v/2025/05/13/kubeflow-cluster-report-html-54j6v-system-container-driver-1370600769"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"CrashLoopBackOff,Failed","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash,error","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \"crash,error\", event_keywords: str = \"CrashLoopBackOff,Failed\"):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_keywords_list = [k.strip() for k in log_keywords.split(\",\")]\n event_keywords_list = [k.strip() for k in event_keywords.split(\",\")]\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n def highlight_keywords(text, keywords):\n for k in keywords:\n text = text.replace(k, f'\u003cspan style=\"color:red\"\u003e{k}\u003c/span\u003e')\n return text\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_keywords = set()\n\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\n\n for line in filtered_describe + filtered_logs:\n for k in log_keywords_list:\n if k in line:\n matched_keywords.add(k)\n\n if filtered_describe or filtered_logs:\n keyword_tag = \", \".join(matched_keywords)\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\u003c/h2\u003e\\n\")\n\n if filtered_describe:\n f.write(\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(\"\\n\".join(filtered_describe), log_keywords_list))\n f.write(\"\u003c/pre\u003e\\n\")\n\n if filtered_logs:\n last_logs = filtered_logs[-10:]\n f.write(\"\u003ch3\u003eLogs (last 10 lines)\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(\"\\n\".join(last_logs), log_keywords_list))\n f.write(\"\u003c/pre\u003e\\n\")\n\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\n if filtered_events:\n namespace_events[namespace] = \"\\n\".join(filtered_events)\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch3\u003eEvents in Namespace: `{ns}`\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(events, event_keywords_list))\n f.write(\"\u003c/pre\u003e\u003c/section\u003e\\n\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"182"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","2654f0dc-728a-44ab-86c2-2c90e7b69f77","--dag_execution_id","182","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-54j6v/2025/05/13/kubeflow-cluster-report-html-54j6v-system-container-driver-1370600769"}}}"defaultValue": "CrashLoopBackOff,Failed", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \"crash,error\", event_keywords: str = \"CrashLoopBackOff,Failed\"):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_keywords_list = [k.strip() for k in log_keywords.split(\",\")]\n event_keywords_list = [k.strip() for k in event_keywords.split(\",\")]\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n def highlight_keywords(text, keywords):\n for k in keywords:\n text = text.replace(k, f'\u003cspan style=\"color:red\"\u003e{k}\u003c/span\u003e')\n return text\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_keywords = set()\n\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\n\n for line in filtered_describe + filtered_logs:\n for k in log_keywords_list:\n if k in line:\n matched_keywords.add(k)\n\n if filtered_describe or filtered_logs:\n keyword_tag = \", \".join(matched_keywords)\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\u003c/h2\u003e\\n\")\n\n if filtered_describe:\n f.write(\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(\"\\n\".join(filtered_describe), log_keywords_list))\n f.write(\"\u003c/pre\u003e\\n\")\n\n if filtered_logs:\n last_logs = filtered_logs[-10:]\n f.write(\"\u003ch3\u003eLogs (last 10 lines)\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(\"\\n\".join(last_logs), log_keywords_list))\n f.write(\"\u003c/pre\u003e\\n\")\n\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\n if filtered_events:\n namespace_events[namespace] = \"\\n\".join(filtered_events)\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch3\u003eEvents in Namespace: `{ns}`\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(events, event_keywords_list))\n f.write(\"\u003c/pre\u003e\u003c/section\u003e\\n\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 05:39:01.790453 14 driver.go:252] parent DAG: id:182 name:"run/2654f0dc-728a-44ab-86c2-2c90e7b69f77" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747114731817 last_update_time_since_epoch:1747114731817 I0513 05:39:01.833484 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"CrashLoopBackOff,Failed,Unhealthy" github_token:string_value:"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" log_keywords:string_value:"crash,error,fail" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 05:39:01.929112 14 driver.go:310] Created execution: id:183 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"92d7d1ad841bc975059c3e84658e2bca3d2f82de556bb46388424a42363b0d63"}} custom_properties:{key:"cached_execution_id" value:{string_value:"180"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:182}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747114741887 last_update_time_since_epoch:1747114741887 "event_keywords": "CrashLoopBackOff,Failed,Unhealthy",
I0513 05:39:11.859863 15 driver.go:252] parent DAG: id:182 name:"run/2654f0dc-728a-44ab-86c2-2c90e7b69f77" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747114731817 last_update_time_since_epoch:1747114731817
I0513 05:39:11.910575 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"CrashLoopBackOff,Failed,Unhealthy" github_token:string_value:"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" log_keywords:string_value:"crash,error,fail" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","2654f0dc-728a-44ab-86c2-2c90e7b69f77","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-54j6v/2025/05/13/kubeflow-cluster-report-html-54j6v-system-dag-driver-809292929"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","2654f0dc-728a-44ab-86c2-2c90e7b69f77","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-54j6v/2025/05/13/kubeflow-cluster-report-html-54j6v-system-dag-driver-809292929"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"CrashLoopBackOff,Failed,Unhealthy","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash,error,fail","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"CrashLoopBackOff,Failed,Unhealthy","github_token":"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","log_keywords":"crash,error,fail","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","2654f0dc-728a-44ab-86c2-2c90e7b69f77","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-54j6v/2025/05/13/kubeflow-cluster-report-html-54j6v-system-dag-driver-809292929"}}}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy", "event_keywords": "CrashLoopBackOff,Failed,Unhealthy", I0513 05:38:51.847686 15 driver.go:185] Created execution: id:182 name:"run/2654f0dc-728a-44ab-86c2-2c90e7b69f77" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747114731817 last_update_time_since_epoch:1747114731817
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"173"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","b4552194-b316-49fe-8ade-36ee4ec911fb","--dag_execution_id","173","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-dvvqs/2025/05/13/kubeflow-cluster-report-html-dvvqs-system-container-driver-2270300684"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"173"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","b4552194-b316-49fe-8ade-36ee4ec911fb","--dag_execution_id","173","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-dvvqs/2025/05/13/kubeflow-cluster-report-html-dvvqs-system-container-driver-2270300684"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"CrashLoopBackOff","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \"crash\", event_keywords: str = \"CrashLoopBackOff\"):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 안전한 keyword 파싱\n log_keys = [k.strip() for k in log_keywords.split(\",\")]\n event_keys = [k.strip() for k in event_keywords.split(\",\")]\n\n # kubectl 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\n h1 {{ color: #2c3e50; }}\n h2 {{ color: #34495e; }}\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\n section {{ margin-bottom: 30px; }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\n matched_logs = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003c/h2\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (Last 10 lines)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch3\u003eEvents in Namespace: {ns}\u003c/h3\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"173"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","b4552194-b316-49fe-8ade-36ee4ec911fb","--dag_execution_id","173","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-dvvqs/2025/05/13/kubeflow-cluster-report-html-dvvqs-system-container-driver-2270300684"}}}"defaultValue": "CrashLoopBackOff", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \"crash\", event_keywords: str = \"CrashLoopBackOff\"):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 안전한 keyword 파싱\n log_keys = [k.strip() for k in log_keywords.split(\",\")]\n event_keys = [k.strip() for k in event_keywords.split(\",\")]\n\n # kubectl 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\n h1 {{ color: #2c3e50; }}\n h2 {{ color: #34495e; }}\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\n section {{ margin-bottom: 30px; }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\n matched_logs = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003c/h2\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (Last 10 lines)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch3\u003eEvents in Namespace: {ns}\u003c/h3\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 04:58:45.393955 15 driver.go:252] parent DAG: id:173 name:"run/b4552194-b316-49fe-8ade-36ee4ec911fb" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747112315432 last_update_time_since_epoch:1747112315432 I0513 04:58:45.434345 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"CrashLoopBackOff,Failed,Unhealthy" github_token:string_value:"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" log_keywords:string_value:"crash,error,fail" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 04:58:45.519265 15 driver.go:310] Created execution: id:174 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"99919ba4da8dc7bd0e660df3cb154109cd1edad72826b01f9599f7571d5efbcb"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:173}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747112325484 last_update_time_since_epoch:1747112325484 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","kubeflow-cluster-report-html","--run_id","b4552194-b316-49fe-8ade-36ee4ec911fb","--execution_id","174","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\", \"log_keywords\":\"crash,error,fail\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/b4552194-b316-49fe-8ade-36ee4ec911fb/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"crash\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \"crash\", event_keywords: str = \"CrashLoopBackOff\"):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 안전한 keyword 파싱\n log_keys = [k.strip() for k in log_keywords.split(\",\")]\n event_keys = [k.strip() for k in event_keywords.split(\",\")]\n\n # kubectl 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\n h1 {{ color: #2c3e50; }}\n h2 {{ color: #34495e; }}\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\n section {{ margin-bottom: 30px; }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\n matched_logs = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003c/h2\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (Last 10 lines)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch3\u003eEvents in Namespace: {ns}\u003c/h3\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "event_keywords": "CrashLoopBackOff,Failed,Unhealthy",
I0513 04:59:57.959316 15 driver.go:252] parent DAG: id:173 name:"run/b4552194-b316-49fe-8ade-36ee4ec911fb" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747112315432 last_update_time_since_epoch:1747112315432
I0513 04:59:58.000518 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"CrashLoopBackOff,Failed,Unhealthy" github_token:string_value:"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" log_keywords:string_value:"crash,error,fail" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"b4552194-b316-49fe-8ade-36ee4ec911fb\",\"--execution_id\",\"174\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/b4552194-b316-49fe-8ade-36ee4ec911fb/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-dvvqs/2025/05/13/kubeflow-cluster-report-html-dvvqs-system-container-impl-1230814434"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"b4552194-b316-49fe-8ade-36ee4ec911fb\",\"--execution_id\",\"174\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/b4552194-b316-49fe-8ade-36ee4ec911fb/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"b4552194-b316-49fe-8ade-36ee4ec911fb\",\"--execution_id\",\"174\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/b4552194-b316-49fe-8ade-36ee4ec911fb/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-dvvqs/2025/05/13/kubeflow-cluster-report-html-dvvqs-system-container-impl-1230814434"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"b4552194-b316-49fe-8ade-36ee4ec911fb\",\"--execution_id\",\"174\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/b4552194-b316-49fe-8ade-36ee4ec911fb/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"b4552194-b316-49fe-8ade-36ee4ec911fb\",\"--execution_id\",\"174\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/b4552194-b316-49fe-8ade-36ee4ec911fb/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-dvvqs/2025/05/13/kubeflow-cluster-report-html-dvvqs-system-container-impl-1230814434"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"b4552194-b316-49fe-8ade-36ee4ec911fb\",\"--execution_id\",\"174\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/b4552194-b316-49fe-8ade-36ee4ec911fb/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"CrashLoopBackOff,Failed,Unhealthy", "log_keywords":"crash,error,fail"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/b4552194-b316-49fe-8ade-36ee4ec911fb/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"crash", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
def collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = "crash", event_keywords: str = "CrashLoopBackOff"):
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"b4552194-b316-49fe-8ade-36ee4ec911fb\",\"--execution_id\",\"174\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/b4552194-b316-49fe-8ade-36ee4ec911fb/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-dvvqs/2025/05/13/kubeflow-cluster-report-html-dvvqs-system-container-impl-1230814434"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"b4552194-b316-49fe-8ade-36ee4ec911fb\",\"--execution_id\",\"174\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/b4552194-b316-49fe-8ade-36ee4ec911fb/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff", "event_keywords": "CrashLoopBackOff,Failed,Unhealthy",
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","b4552194-b316-49fe-8ade-36ee4ec911fb","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-dvvqs/2025/05/13/kubeflow-cluster-report-html-dvvqs-system-dag-driver-1885421972"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","b4552194-b316-49fe-8ade-36ee4ec911fb","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-dvvqs/2025/05/13/kubeflow-cluster-report-html-dvvqs-system-dag-driver-1885421972"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"CrashLoopBackOff,Failed,Unhealthy","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash,error,fail","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"CrashLoopBackOff,Failed,Unhealthy","github_token":"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","log_keywords":"crash,error,fail","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","b4552194-b316-49fe-8ade-36ee4ec911fb","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-dvvqs/2025/05/13/kubeflow-cluster-report-html-dvvqs-system-dag-driver-1885421972"}}}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy", "event_keywords": "CrashLoopBackOff,Failed,Unhealthy", I0513 04:58:35.476506 14 driver.go:185] Created execution: id:173 name:"run/b4552194-b316-49fe-8ade-36ee4ec911fb" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747112315432 last_update_time_since_epoch:1747112315432
I0513 05:38:07.513534 14 driver.go:252] parent DAG: id:179 name:"run/72f85b13-5ff0-48ac-8889-032ba3f822b0" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747114603019 last_update_time_since_epoch:1747114603019
I0513 05:38:07.556069 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"CrashLoopBackOff,Failed,Unhealthy" github_token:string_value:"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" log_keywords:string_value:"crash,error,fail" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"179"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","72f85b13-5ff0-48ac-8889-032ba3f822b0","--dag_execution_id","179","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-gpnjw/2025/05/13/kubeflow-cluster-report-html-gpnjw-system-container-driver-3716821788"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"179"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","72f85b13-5ff0-48ac-8889-032ba3f822b0","--dag_execution_id","179","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-gpnjw/2025/05/13/kubeflow-cluster-report-html-gpnjw-system-container-driver-3716821788"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"CrashLoopBackOff,Failed","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash,error","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \"crash,error\", event_keywords: str = \"CrashLoopBackOff,Failed\"):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_keywords_list = [k.strip() for k in log_keywords.split(\",\")]\n event_keywords_list = [k.strip() for k in event_keywords.split(\",\")]\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n def highlight_keywords(text, keywords):\n for k in keywords:\n text = text.replace(k, f'\u003cspan style=\"color:red\"\u003e{k}\u003c/span\u003e')\n return text\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_keywords = set()\n\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\n\n for line in filtered_describe + filtered_logs:\n for k in log_keywords_list:\n if k in line:\n matched_keywords.add(k)\n\n if filtered_describe or filtered_logs:\n keyword_tag = \", \".join(matched_keywords)\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\u003c/h2\u003e\\n\")\n\n if filtered_describe:\n f.write(\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(\"\\n\".join(filtered_describe), log_keywords_list))\n f.write(\"\u003c/pre\u003e\\n\")\n\n if filtered_logs:\n last_logs = filtered_logs[-10:]\n f.write(\"\u003ch3\u003eLogs (last 10 lines)\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(\"\\n\".join(last_logs), log_keywords_list))\n f.write(\"\u003c/pre\u003e\\n\")\n\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\n if filtered_events:\n namespace_events[namespace] = \"\\n\".join(filtered_events)\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch3\u003eEvents in Namespace: `{ns}`\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(events, event_keywords_list))\n f.write(\"\u003c/pre\u003e\u003c/section\u003e\\n\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"179"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","72f85b13-5ff0-48ac-8889-032ba3f822b0","--dag_execution_id","179","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-gpnjw/2025/05/13/kubeflow-cluster-report-html-gpnjw-system-container-driver-3716821788"}}}"defaultValue": "CrashLoopBackOff,Failed", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \"crash,error\", event_keywords: str = \"CrashLoopBackOff,Failed\"):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_keywords_list = [k.strip() for k in log_keywords.split(\",\")]\n event_keywords_list = [k.strip() for k in event_keywords.split(\",\")]\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n def highlight_keywords(text, keywords):\n for k in keywords:\n text = text.replace(k, f'\u003cspan style=\"color:red\"\u003e{k}\u003c/span\u003e')\n return text\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_keywords = set()\n\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\n\n for line in filtered_describe + filtered_logs:\n for k in log_keywords_list:\n if k in line:\n matched_keywords.add(k)\n\n if filtered_describe or filtered_logs:\n keyword_tag = \", \".join(matched_keywords)\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\u003c/h2\u003e\\n\")\n\n if filtered_describe:\n f.write(\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(\"\\n\".join(filtered_describe), log_keywords_list))\n f.write(\"\u003c/pre\u003e\\n\")\n\n if filtered_logs:\n last_logs = filtered_logs[-10:]\n f.write(\"\u003ch3\u003eLogs (last 10 lines)\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(\"\\n\".join(last_logs), log_keywords_list))\n f.write(\"\u003c/pre\u003e\\n\")\n\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\n if filtered_events:\n namespace_events[namespace] = \"\\n\".join(filtered_events)\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch3\u003eEvents in Namespace: `{ns}`\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(events, event_keywords_list))\n f.write(\"\u003c/pre\u003e\u003c/section\u003e\\n\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 05:36:52.053234 15 driver.go:252] parent DAG: id:179 name:"run/72f85b13-5ff0-48ac-8889-032ba3f822b0" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747114603019 last_update_time_since_epoch:1747114603019 I0513 05:36:52.093757 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"CrashLoopBackOff,Failed,Unhealthy" github_token:string_value:"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" log_keywords:string_value:"crash,error,fail" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 05:36:52.172093 15 driver.go:310] Created execution: id:180 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"92d7d1ad841bc975059c3e84658e2bca3d2f82de556bb46388424a42363b0d63"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:179}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747114612139 last_update_time_since_epoch:1747114612139 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","kubeflow-cluster-report-html","--run_id","72f85b13-5ff0-48ac-8889-032ba3f822b0","--execution_id","180","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\", \"log_keywords\":\"crash,error,fail\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/72f85b13-5ff0-48ac-8889-032ba3f822b0/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,Failed\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"crash,error\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \"crash,error\", event_keywords: str = \"CrashLoopBackOff,Failed\"):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_keywords_list = [k.strip() for k in log_keywords.split(\",\")]\n event_keywords_list = [k.strip() for k in event_keywords.split(\",\")]\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n def highlight_keywords(text, keywords):\n for k in keywords:\n text = text.replace(k, f'\u003cspan style=\"color:red\"\u003e{k}\u003c/span\u003e')\n return text\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_keywords = set()\n\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\n\n for line in filtered_describe + filtered_logs:\n for k in log_keywords_list:\n if k in line:\n matched_keywords.add(k)\n\n if filtered_describe or filtered_logs:\n keyword_tag = \", \".join(matched_keywords)\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\u003c/h2\u003e\\n\")\n\n if filtered_describe:\n f.write(\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(\"\\n\".join(filtered_describe), log_keywords_list))\n f.write(\"\u003c/pre\u003e\\n\")\n\n if filtered_logs:\n last_logs = filtered_logs[-10:]\n f.write(\"\u003ch3\u003eLogs (last 10 lines)\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(\"\\n\".join(last_logs), log_keywords_list))\n f.write(\"\u003c/pre\u003e\\n\")\n\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\n if filtered_events:\n namespace_events[namespace] = \"\\n\".join(filtered_events)\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch3\u003eEvents in Namespace: `{ns}`\u003c/h3\u003e\u003cpre\u003e\")\n f.write(highlight_keywords(events, event_keywords_list))\n f.write(\"\u003c/pre\u003e\u003c/section\u003e\\n\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "event_keywords": "CrashLoopBackOff,Failed,Unhealthy",
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"72f85b13-5ff0-48ac-8889-032ba3f822b0\",\"--execution_id\",\"180\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/72f85b13-5ff0-48ac-8889-032ba3f822b0/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-gpnjw/2025/05/13/kubeflow-cluster-report-html-gpnjw-system-container-impl-4149860178"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"72f85b13-5ff0-48ac-8889-032ba3f822b0\",\"--execution_id\",\"180\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/72f85b13-5ff0-48ac-8889-032ba3f822b0/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"72f85b13-5ff0-48ac-8889-032ba3f822b0\",\"--execution_id\",\"180\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/72f85b13-5ff0-48ac-8889-032ba3f822b0/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-gpnjw/2025/05/13/kubeflow-cluster-report-html-gpnjw-system-container-impl-4149860178"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"72f85b13-5ff0-48ac-8889-032ba3f822b0\",\"--execution_id\",\"180\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/72f85b13-5ff0-48ac-8889-032ba3f822b0/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"72f85b13-5ff0-48ac-8889-032ba3f822b0\",\"--execution_id\",\"180\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/72f85b13-5ff0-48ac-8889-032ba3f822b0/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-gpnjw/2025/05/13/kubeflow-cluster-report-html-gpnjw-system-container-impl-4149860178"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"72f85b13-5ff0-48ac-8889-032ba3f822b0\",\"--execution_id\",\"180\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/72f85b13-5ff0-48ac-8889-032ba3f822b0/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"CrashLoopBackOff,Failed,Unhealthy", "log_keywords":"crash,error,fail"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/72f85b13-5ff0-48ac-8889-032ba3f822b0/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,Failed", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"crash,error", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
def collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = "crash,error", event_keywords: str = "CrashLoopBackOff,Failed"):
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"72f85b13-5ff0-48ac-8889-032ba3f822b0\",\"--execution_id\",\"180\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/72f85b13-5ff0-48ac-8889-032ba3f822b0/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-gpnjw/2025/05/13/kubeflow-cluster-report-html-gpnjw-system-container-impl-4149860178"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"kubeflow-cluster-report-html\",\"--run_id\",\"72f85b13-5ff0-48ac-8889-032ba3f822b0\",\"--execution_id\",\"180\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"CrashLoopBackOff,Failed,Unhealthy\\\", \\\"log_keywords\\\":\\\"crash,error,fail\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/kubeflow-cluster-report-html/72f85b13-5ff0-48ac-8889-032ba3f822b0/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash,error\\\", event_keywords: str = \\\"CrashLoopBackOff,Failed\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_keywords_list = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keywords_list = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def highlight_keywords(text, keywords):\\n for k in keywords:\\n text = text.replace(k, f'\\u003cspan style=\\\"color:red\\\"\\u003e{k}\\u003c/span\\u003e')\\n return text\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_keywords = set()\\n\\n filtered_describe = [l for l in describe.splitlines() if any(k in l for k in log_keywords_list)]\\n filtered_logs = [l for l in logs.splitlines() if any(k in l for k in log_keywords_list)]\\n\\n for line in filtered_describe + filtered_logs:\\n for k in log_keywords_list:\\n if k in line:\\n matched_keywords.add(k)\\n\\n if filtered_describe or filtered_logs:\\n keyword_tag = \\\", \\\".join(matched_keywords)\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: `{pod_name}` (Namespace: `{namespace}`) [{keyword_tag}]\\u003c/h2\\u003e\\\\n\\\")\\n\\n if filtered_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(filtered_describe), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n if filtered_logs:\\n last_logs = filtered_logs[-10:]\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(\\\"\\\\n\\\".join(last_logs), log_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n filtered_events = [l for l in events.splitlines() if any(k in l for k in event_keywords_list)]\\n if filtered_events:\\n namespace_events[namespace] = \\\"\\\\n\\\".join(filtered_events)\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: `{ns}`\\u003c/h3\\u003e\\u003cpre\\u003e\\\")\\n f.write(highlight_keywords(events, event_keywords_list))\\n f.write(\\\"\\u003c/pre\\u003e\\u003c/section\\u003e\\\\n\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,Failed", "event_keywords": "CrashLoopBackOff,Failed,Unhealthy",
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","72f85b13-5ff0-48ac-8889-032ba3f822b0","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-gpnjw/2025/05/13/kubeflow-cluster-report-html-gpnjw-system-dag-driver-2435536036"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","72f85b13-5ff0-48ac-8889-032ba3f822b0","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-gpnjw/2025/05/13/kubeflow-cluster-report-html-gpnjw-system-dag-driver-2435536036"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"CrashLoopBackOff,Failed,Unhealthy","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash,error,fail","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"CrashLoopBackOff,Failed,Unhealthy","github_token":"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","log_keywords":"crash,error,fail","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","72f85b13-5ff0-48ac-8889-032ba3f822b0","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-gpnjw/2025/05/13/kubeflow-cluster-report-html-gpnjw-system-dag-driver-2435536036"}}}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy", "event_keywords": "CrashLoopBackOff,Failed,Unhealthy", I0513 05:36:43.050232 14 driver.go:185] Created execution: id:179 name:"run/72f85b13-5ff0-48ac-8889-032ba3f822b0" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747114603019 last_update_time_since_epoch:1747114603019
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"176"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","03166a1b-4161-40e6-92ea-c0452fb97123","--dag_execution_id","176","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-lstxq/2025/05/13/kubeflow-cluster-report-html-lstxq-system-container-driver-3997409308"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"176"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","03166a1b-4161-40e6-92ea-c0452fb97123","--dag_execution_id","176","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-lstxq/2025/05/13/kubeflow-cluster-report-html-lstxq-system-container-driver-3997409308"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"CrashLoopBackOff","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \"crash\", event_keywords: str = \"CrashLoopBackOff\"):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 안전한 keyword 파싱\n log_keys = [k.strip() for k in log_keywords.split(\",\")]\n event_keys = [k.strip() for k in event_keywords.split(\",\")]\n\n # kubectl 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\n h1 {{ color: #2c3e50; }}\n h2 {{ color: #34495e; }}\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\n section {{ margin-bottom: 30px; }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\n matched_logs = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003c/h2\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (Last 10 lines)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch3\u003eEvents in Namespace: {ns}\u003c/h3\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"176"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","kubeflow-cluster-report-html","--run_id","03166a1b-4161-40e6-92ea-c0452fb97123","--dag_execution_id","176","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \\\"crash\\\", event_keywords: str = \\\"CrashLoopBackOff\\\"):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 안전한 keyword 파싱\\n log_keys = [k.strip() for k in log_keywords.split(\\\",\\\")]\\n event_keys = [k.strip() for k in event_keywords.split(\\\",\\\")]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\", encoding=\\\"utf-8\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\\n h1 {{ color: #2c3e50; }}\\n h2 {{ color: #34495e; }}\\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\\n section {{ margin-bottom: 30px; }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\\n matched_logs = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (Last 10 lines)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch3\\u003eEvents in Namespace: {ns}\\u003c/h3\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-lstxq/2025/05/13/kubeflow-cluster-report-html-lstxq-system-container-driver-3997409308"}}}"defaultValue": "CrashLoopBackOff", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset], log_keywords: str = \"crash\", event_keywords: str = \"CrashLoopBackOff\"):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 안전한 keyword 파싱\n log_keys = [k.strip() for k in log_keywords.split(\",\")]\n event_keys = [k.strip() for k in event_keywords.split(\",\")]\n\n # kubectl 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }}\n h1 {{ color: #2c3e50; }}\n h2 {{ color: #34495e; }}\n pre {{ background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }}\n section {{ margin-bottom: 30px; }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_keys)])\n matched_logs = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_keys)][-10:]) # 마지막 10줄\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003c/h2\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (Last 10 lines)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_keys)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch3\u003eEvents in Namespace: {ns}\u003c/h3\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 05:12:37.696302 13 driver.go:252] parent DAG: id:176 name:"run/03166a1b-4161-40e6-92ea-c0452fb97123" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747113147727 last_update_time_since_epoch:1747113147727 I0513 05:12:37.741985 13 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"CrashLoopBackOff,Failed,Unhealthy" github_token:string_value:"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" log_keywords:string_value:"crash,error,fail" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 05:12:37.828114 13 driver.go:310] Created execution: id:177 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"99919ba4da8dc7bd0e660df3cb154109cd1edad72826b01f9599f7571d5efbcb"}} custom_properties:{key:"cached_execution_id" value:{string_value:"174"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:176}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747113157789 last_update_time_since_epoch:1747113157789 "event_keywords": "CrashLoopBackOff,Failed,Unhealthy",
I0513 05:12:47.726821 15 driver.go:252] parent DAG: id:176 name:"run/03166a1b-4161-40e6-92ea-c0452fb97123" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747113147727 last_update_time_since_epoch:1747113147727
I0513 05:12:47.770168 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"CrashLoopBackOff,Failed,Unhealthy" github_token:string_value:"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" log_keywords:string_value:"crash,error,fail" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","03166a1b-4161-40e6-92ea-c0452fb97123","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-lstxq/2025/05/13/kubeflow-cluster-report-html-lstxq-system-dag-driver-1883122084"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","03166a1b-4161-40e6-92ea-c0452fb97123","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-lstxq/2025/05/13/kubeflow-cluster-report-html-lstxq-system-dag-driver-1883122084"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"CrashLoopBackOff,Failed,Unhealthy","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash,error,fail","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"CrashLoopBackOff,Failed,Unhealthy","github_token":"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","log_keywords":"crash,error,fail","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","kubeflow-cluster-report-html","--run_id","03166a1b-4161-40e6-92ea-c0452fb97123","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"CrashLoopBackOff,Failed,Unhealthy\",\"github_token\":\"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\",\"log_keywords\":\"crash,error,fail\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/kubeflow-cluster-report-html-lstxq/2025/05/13/kubeflow-cluster-report-html-lstxq-system-dag-driver-1883122084"}}}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy", "event_keywords": "CrashLoopBackOff,Failed,Unhealthy", I0513 05:12:27.758632 15 driver.go:185] Created execution: id:176 name:"run/03166a1b-4161-40e6-92ea-c0452fb97123" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"CrashLoopBackOff,Failed,Unhealthy"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"crash,error,fail"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747113147727 last_update_time_since_epoch:1747113147727
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"185"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","03e32b33-5d23-4ce5-be06-00812f5b167b","--dag_execution_id","185","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-4ghtw/2025/05/13/pod-log-pipeline-advance-version-4ghtw-system-container-driver-1435614314"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"185"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","03e32b33-5d23-4ce5-be06-00812f5b167b","--dag_execution_id","185","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-4ghtw/2025/05/13/pod-log-pipeline-advance-version-4ghtw-system-container-driver-1435614314"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"185"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","03e32b33-5d23-4ce5-be06-00812f5b167b","--dag_execution_id","185","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-4ghtw/2025/05/13/pod-log-pipeline-advance-version-4ghtw-system-container-driver-1435614314"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 06:05:33.768168 14 driver.go:252] parent DAG: id:185 name:"run/03e32b33-5d23-4ce5-be06-00812f5b167b" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747116323803 last_update_time_since_epoch:1747116323803 I0513 06:05:33.807352 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 06:05:33.887845 14 driver.go:310] Created execution: id:186 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"f7d2838b294b9826fe0e2fb40d5b20cfcb6d695d9cd08eddd83eed493416dc60"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:185}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747116333854 last_update_time_since_epoch:1747116333854 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","03e32b33-5d23-4ce5-be06-00812f5b167b","--execution_id","186","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/03e32b33-5d23-4ce5-be06-00812f5b167b/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
I0513 06:06:34.197660 14 driver.go:252] parent DAG: id:185 name:"run/03e32b33-5d23-4ce5-be06-00812f5b167b" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747116323803 last_update_time_since_epoch:1747116323803
I0513 06:06:34.244558 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"03e32b33-5d23-4ce5-be06-00812f5b167b\",\"--execution_id\",\"186\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/03e32b33-5d23-4ce5-be06-00812f5b167b/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-4ghtw/2025/05/13/pod-log-pipeline-advance-version-4ghtw-system-container-impl-2336668228"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"03e32b33-5d23-4ce5-be06-00812f5b167b\",\"--execution_id\",\"186\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/03e32b33-5d23-4ce5-be06-00812f5b167b/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"03e32b33-5d23-4ce5-be06-00812f5b167b\",\"--execution_id\",\"186\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/03e32b33-5d23-4ce5-be06-00812f5b167b/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-4ghtw/2025/05/13/pod-log-pipeline-advance-version-4ghtw-system-container-impl-2336668228"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"03e32b33-5d23-4ce5-be06-00812f5b167b\",\"--execution_id\",\"186\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/03e32b33-5d23-4ce5-be06-00812f5b167b/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"03e32b33-5d23-4ce5-be06-00812f5b167b\",\"--execution_id\",\"186\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/03e32b33-5d23-4ce5-be06-00812f5b167b/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-4ghtw/2025/05/13/pod-log-pipeline-advance-version-4ghtw-system-container-impl-2336668228"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"03e32b33-5d23-4ce5-be06-00812f5b167b\",\"--execution_id\",\"186\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/03e32b33-5d23-4ce5-be06-00812f5b167b/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/03e32b33-5d23-4ce5-be06-00812f5b167b/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready",
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"03e32b33-5d23-4ce5-be06-00812f5b167b\",\"--execution_id\",\"186\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/03e32b33-5d23-4ce5-be06-00812f5b167b/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-4ghtw/2025/05/13/pod-log-pipeline-advance-version-4ghtw-system-container-impl-2336668228"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"03e32b33-5d23-4ce5-be06-00812f5b167b\",\"--execution_id\",\"186\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/03e32b33-5d23-4ce5-be06-00812f5b167b/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","03e32b33-5d23-4ce5-be06-00812f5b167b","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-4ghtw/2025/05/13/pod-log-pipeline-advance-version-4ghtw-system-dag-driver-1127347630"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","03e32b33-5d23-4ce5-be06-00812f5b167b","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-4ghtw/2025/05/13/pod-log-pipeline-advance-version-4ghtw-system-dag-driver-1127347630"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","03e32b33-5d23-4ce5-be06-00812f5b167b","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-4ghtw/2025/05/13/pod-log-pipeline-advance-version-4ghtw-system-dag-driver-1127347630"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 06:05:23.833494 15 driver.go:185] Created execution: id:185 name:"run/03e32b33-5d23-4ce5-be06-00812f5b167b" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747116323803 last_update_time_since_epoch:1747116323803
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"206"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","64034148-0e9a-464b-a9aa-f5dd9da7fbfa","--dag_execution_id","206","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5g5dq/2025/05/13/pod-log-pipeline-advance-version-5g5dq-system-container-driver-2182329916"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"206"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","64034148-0e9a-464b-a9aa-f5dd9da7fbfa","--dag_execution_id","206","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5g5dq/2025/05/13/pod-log-pipeline-advance-version-5g5dq-system-container-driver-2182329916"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"206"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","64034148-0e9a-464b-a9aa-f5dd9da7fbfa","--dag_execution_id","206","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5g5dq/2025/05/13/pod-log-pipeline-advance-version-5g5dq-system-container-driver-2182329916"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 07:36:25.576165 15 driver.go:252] parent DAG: id:206 name:"run/64034148-0e9a-464b-a9aa-f5dd9da7fbfa" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747121775625 last_update_time_since_epoch:1747121775625 I0513 07:36:25.621424 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 07:36:25.702303 15 driver.go:310] Created execution: id:207 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"ec7357b8454fa3705a6ec901b80973fd1d4bf2e3f469a60a99bae7e3b4566eda"}} custom_properties:{key:"cached_execution_id" value:{string_value:"196"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:206}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747121785667 last_update_time_since_epoch:1747121785667 "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
I0513 07:36:35.771781 15 driver.go:252] parent DAG: id:206 name:"run/64034148-0e9a-464b-a9aa-f5dd9da7fbfa" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747121775625 last_update_time_since_epoch:1747121775625
I0513 07:36:35.811967 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","64034148-0e9a-464b-a9aa-f5dd9da7fbfa","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5g5dq/2025/05/13/pod-log-pipeline-advance-version-5g5dq-system-dag-driver-3484045636"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","64034148-0e9a-464b-a9aa-f5dd9da7fbfa","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5g5dq/2025/05/13/pod-log-pipeline-advance-version-5g5dq-system-dag-driver-3484045636"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","64034148-0e9a-464b-a9aa-f5dd9da7fbfa","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5g5dq/2025/05/13/pod-log-pipeline-advance-version-5g5dq-system-dag-driver-3484045636"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 07:36:15.655204 15 driver.go:185] Created execution: id:206 name:"run/64034148-0e9a-464b-a9aa-f5dd9da7fbfa" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747121775625 last_update_time_since_epoch:1747121775625
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"232"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","2c4a0994-92f0-4682-9289-dda2e308be96","--dag_execution_id","232","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-driver-2407325076"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"232"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","2c4a0994-92f0-4682-9289-dda2e308be96","--dag_execution_id","232","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-driver-2407325076"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash,error,fail,oomkilled,evicted","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\n h2 {{ color: #34495e; margin-top: 40px; }}\n h3 {{ margin-top: 20px; color: #2e86de; }}\n pre {{\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }}\n section {{ margin-bottom: 40px; }}\n .highlight {{\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"232"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","2c4a0994-92f0-4682-9289-dda2e308be96","--dag_execution_id","232","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-driver-2407325076"}}}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "defaultValue": "crash,error,fail,oomkilled,evicted", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\n h2 {{ color: #34495e; margin-top: 40px; }}\n h3 {{ margin-top: 20px; color: #2e86de; }}\n pre {{\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }}\n section {{ margin-bottom: 40px; }}\n .highlight {{\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0514 00:23:53.690361 15 driver.go:252] parent DAG: id:232 name:"run/2c4a0994-92f0-4682-9289-dda2e308be96" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747182223538 last_update_time_since_epoch:1747182223538 I0514 00:23:53.732342 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0514 00:23:53.804460 15 driver.go:310] Created execution: id:233 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:232}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747182233770 last_update_time_since_epoch:1747182233770 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","2c4a0994-92f0-4682-9289-dda2e308be96","--execution_id","233","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"crash,error,fail,oomkilled,evicted\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\n h2 {{ color: #34495e; margin-top: 40px; }}\n h3 {{ margin-top: 20px; color: #2e86de; }}\n pre {{\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }}\n section {{ margin-bottom: 40px; }}\n .highlight {{\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
I0514 00:25:25.712558 14 driver.go:252] parent DAG: id:232 name:"run/2c4a0994-92f0-4682-9289-dda2e308be96" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747182223538 last_update_time_since_epoch:1747182223538
I0514 00:25:25.757985 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--execution_id\",\"233\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-impl-4123328826"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--execution_id\",\"233\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--execution_id\",\"233\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-impl-4123328826"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--execution_id\",\"233\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--execution_id\",\"233\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-impl-4123328826"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--execution_id\",\"233\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"crash,error,fail,oomkilled,evicted", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "crash,error,fail,oomkilled,evicted",
event_keywords: str = "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled"):
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--execution_id\",\"233\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-impl-4123328826"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--execution_id\",\"233\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "defaultValue": "crash,error,fail,oomkilled,evicted", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","2c4a0994-92f0-4682-9289-dda2e308be96","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-dag-driver-92227836"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","2c4a0994-92f0-4682-9289-dda2e308be96","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-dag-driver-92227836"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","2c4a0994-92f0-4682-9289-dda2e308be96","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-dag-driver-92227836"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0514 00:23:43.571922 15 driver.go:185] Created execution: id:232 name:"run/2c4a0994-92f0-4682-9289-dda2e308be96" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747182223538 last_update_time_since_epoch:1747182223538
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"213"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","e1569055-0952-4b02-b449-45911339fa78","--dag_execution_id","213","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-7nshw/2025/05/13/pod-log-pipeline-advance-version-7nshw-system-container-driver-2969399015"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"213"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","e1569055-0952-4b02-b449-45911339fa78","--dag_execution_id","213","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-7nshw/2025/05/13/pod-log-pipeline-advance-version-7nshw-system-container-driver-2969399015"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"213"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","e1569055-0952-4b02-b449-45911339fa78","--dag_execution_id","213","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-7nshw/2025/05/13/pod-log-pipeline-advance-version-7nshw-system-container-driver-2969399015"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 08:19:56.685939 15 driver.go:252] parent DAG: id:213 name:"run/e1569055-0952-4b02-b449-45911339fa78" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747124386690 last_update_time_since_epoch:1747124386690 I0513 08:19:56.727541 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 08:19:56.801733 15 driver.go:310] Created execution: id:214 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"Collect Filtered Logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:213}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"Collect Filtered Logs"}} create_time_since_epoch:1747124396764 last_update_time_since_epoch:1747124396764 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","e1569055-0952-4b02-b449-45911339fa78","--execution_id","214","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/e1569055-0952-4b02-b449-45911339fa78/Collect Filtered Logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
I0513 08:21:34.587747 14 driver.go:252] parent DAG: id:213 name:"run/e1569055-0952-4b02-b449-45911339fa78" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747124386690 last_update_time_since_epoch:1747124386690
I0513 08:21:34.633972 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"e1569055-0952-4b02-b449-45911339fa78\",\"--execution_id\",\"214\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/e1569055-0952-4b02-b449-45911339fa78/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-7nshw/2025/05/13/pod-log-pipeline-advance-version-7nshw-system-container-impl-1314902985"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"e1569055-0952-4b02-b449-45911339fa78\",\"--execution_id\",\"214\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/e1569055-0952-4b02-b449-45911339fa78/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"e1569055-0952-4b02-b449-45911339fa78\",\"--execution_id\",\"214\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/e1569055-0952-4b02-b449-45911339fa78/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-7nshw/2025/05/13/pod-log-pipeline-advance-version-7nshw-system-container-impl-1314902985"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"e1569055-0952-4b02-b449-45911339fa78\",\"--execution_id\",\"214\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/e1569055-0952-4b02-b449-45911339fa78/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"e1569055-0952-4b02-b449-45911339fa78\",\"--execution_id\",\"214\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/e1569055-0952-4b02-b449-45911339fa78/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-7nshw/2025/05/13/pod-log-pipeline-advance-version-7nshw-system-container-impl-1314902985"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"e1569055-0952-4b02-b449-45911339fa78\",\"--execution_id\",\"214\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/e1569055-0952-4b02-b449-45911339fa78/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/e1569055-0952-4b02-b449-45911339fa78/Collect Filtered Logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready",
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"e1569055-0952-4b02-b449-45911339fa78\",\"--execution_id\",\"214\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/e1569055-0952-4b02-b449-45911339fa78/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-7nshw/2025/05/13/pod-log-pipeline-advance-version-7nshw-system-container-impl-1314902985"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"e1569055-0952-4b02-b449-45911339fa78\",\"--execution_id\",\"214\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/e1569055-0952-4b02-b449-45911339fa78/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","e1569055-0952-4b02-b449-45911339fa78","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-7nshw/2025/05/13/pod-log-pipeline-advance-version-7nshw-system-dag-driver-2469645371"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","e1569055-0952-4b02-b449-45911339fa78","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-7nshw/2025/05/13/pod-log-pipeline-advance-version-7nshw-system-dag-driver-2469645371"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"Collect Filtered Logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","e1569055-0952-4b02-b449-45911339fa78","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-7nshw/2025/05/13/pod-log-pipeline-advance-version-7nshw-system-dag-driver-2469645371"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 08:19:46.726592 14 driver.go:185] Created execution: id:213 name:"run/e1569055-0952-4b02-b449-45911339fa78" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747124386690 last_update_time_since_epoch:1747124386690
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"198"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","5fa5127c-b3b5-4556-94f5-8601cdce43e6","--dag_execution_id","198","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-8lppr/2025/05/13/pod-log-pipeline-advance-version-8lppr-system-container-driver-3978863182"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"198"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","5fa5127c-b3b5-4556-94f5-8601cdce43e6","--dag_execution_id","198","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-8lppr/2025/05/13/pod-log-pipeline-advance-version-8lppr-system-container-driver-3978863182"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"198"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","5fa5127c-b3b5-4556-94f5-8601cdce43e6","--dag_execution_id","198","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-8lppr/2025/05/13/pod-log-pipeline-advance-version-8lppr-system-container-driver-3978863182"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 06:26:48.883964 15 driver.go:252] parent DAG: id:198 name:"run/5fa5127c-b3b5-4556-94f5-8601cdce43e6" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747117598836 last_update_time_since_epoch:1747117598836 I0513 06:26:48.927786 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 06:26:49.013255 15 driver.go:310] Created execution: id:199 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"ec7357b8454fa3705a6ec901b80973fd1d4bf2e3f469a60a99bae7e3b4566eda"}} custom_properties:{key:"cached_execution_id" value:{string_value:"196"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:198}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747117608976 last_update_time_since_epoch:1747117608976 "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
I0513 06:26:58.878307 15 driver.go:252] parent DAG: id:198 name:"run/5fa5127c-b3b5-4556-94f5-8601cdce43e6" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747117598836 last_update_time_since_epoch:1747117598836
I0513 06:26:58.924117 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","5fa5127c-b3b5-4556-94f5-8601cdce43e6","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-8lppr/2025/05/13/pod-log-pipeline-advance-version-8lppr-system-dag-driver-3081928554"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","5fa5127c-b3b5-4556-94f5-8601cdce43e6","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-8lppr/2025/05/13/pod-log-pipeline-advance-version-8lppr-system-dag-driver-3081928554"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","5fa5127c-b3b5-4556-94f5-8601cdce43e6","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-8lppr/2025/05/13/pod-log-pipeline-advance-version-8lppr-system-dag-driver-3081928554"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 06:26:38.868444 13 driver.go:185] Created execution: id:198 name:"run/5fa5127c-b3b5-4556-94f5-8601cdce43e6" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747117598836 last_update_time_since_epoch:1747117598836
I0513 07:55:42.984599 15 driver.go:252] parent DAG: id:209 name:"run/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747122866425 last_update_time_since_epoch:1747122866425
I0513 07:55:43.025492 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"209"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68","--dag_execution_id","209","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-b8cvw/2025/05/13/pod-log-pipeline-advance-version-b8cvw-system-container-driver-720270776"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"209"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68","--dag_execution_id","209","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-b8cvw/2025/05/13/pod-log-pipeline-advance-version-b8cvw-system-container-driver-720270776"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"209"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68","--dag_execution_id","209","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-b8cvw/2025/05/13/pod-log-pipeline-advance-version-b8cvw-system-container-driver-720270776"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 07:54:36.460416 15 driver.go:252] parent DAG: id:209 name:"run/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747122866425 last_update_time_since_epoch:1747122866425 I0513 07:54:36.501347 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 07:54:36.571875 15 driver.go:310] Created execution: id:210 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"Collect Filtered Logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:209}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"Collect Filtered Logs"}} create_time_since_epoch:1747122876539 last_update_time_since_epoch:1747122876539 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68","--execution_id","210","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68/Collect Filtered Logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68\",\"--execution_id\",\"210\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-b8cvw/2025/05/13/pod-log-pipeline-advance-version-b8cvw-system-container-impl-2969120782"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68\",\"--execution_id\",\"210\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68\",\"--execution_id\",\"210\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-b8cvw/2025/05/13/pod-log-pipeline-advance-version-b8cvw-system-container-impl-2969120782"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68\",\"--execution_id\",\"210\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68\",\"--execution_id\",\"210\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-b8cvw/2025/05/13/pod-log-pipeline-advance-version-b8cvw-system-container-impl-2969120782"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68\",\"--execution_id\",\"210\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68/Collect Filtered Logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready",
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68\",\"--execution_id\",\"210\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-b8cvw/2025/05/13/pod-log-pipeline-advance-version-b8cvw-system-container-impl-2969120782"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68\",\"--execution_id\",\"210\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-b8cvw/2025/05/13/pod-log-pipeline-advance-version-b8cvw-system-dag-driver-3183041576"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-b8cvw/2025/05/13/pod-log-pipeline-advance-version-b8cvw-system-dag-driver-3183041576"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"Collect Filtered Logs"}},"upload-to-github":{"cachingOptions":{},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"Upload to GitHub"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-b8cvw/2025/05/13/pod-log-pipeline-advance-version-b8cvw-system-dag-driver-3183041576"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 07:54:26.456852 14 driver.go:185] Created execution: id:209 name:"run/f480cfbe-18f9-4bdb-b0bf-71cf5fa5ae68" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747122866425 last_update_time_since_epoch:1747122866425
I0513 08:30:46.740990 15 driver.go:252] parent DAG: id:215 name:"run/813cdf2b-6ff3-4126-9631-6b3b60112417" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125027682 last_update_time_since_epoch:1747125027682
I0513 08:30:46.783899 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"215"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","813cdf2b-6ff3-4126-9631-6b3b60112417","--dag_execution_id","215","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-cs852/2025/05/13/pod-log-pipeline-advance-version-cs852-system-container-driver-888646629"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"215"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","813cdf2b-6ff3-4126-9631-6b3b60112417","--dag_execution_id","215","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-cs852/2025/05/13/pod-log-pipeline-advance-version-cs852-system-container-driver-888646629"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"215"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","813cdf2b-6ff3-4126-9631-6b3b60112417","--dag_execution_id","215","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-cs852/2025/05/13/pod-log-pipeline-advance-version-cs852-system-container-driver-888646629"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 08:30:36.650507 14 driver.go:252] parent DAG: id:215 name:"run/813cdf2b-6ff3-4126-9631-6b3b60112417" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125027682 last_update_time_since_epoch:1747125027682 I0513 08:30:36.694725 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 08:30:36.778749 14 driver.go:310] Created execution: id:216 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"ec7357b8454fa3705a6ec901b80973fd1d4bf2e3f469a60a99bae7e3b4566eda"}} custom_properties:{key:"cached_execution_id" value:{string_value:"196"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:215}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747125036746 last_update_time_since_epoch:1747125036746 "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","813cdf2b-6ff3-4126-9631-6b3b60112417","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-cs852/2025/05/13/pod-log-pipeline-advance-version-cs852-system-dag-driver-3421844605"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","813cdf2b-6ff3-4126-9631-6b3b60112417","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-cs852/2025/05/13/pod-log-pipeline-advance-version-cs852-system-dag-driver-3421844605"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","813cdf2b-6ff3-4126-9631-6b3b60112417","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-cs852/2025/05/13/pod-log-pipeline-advance-version-cs852-system-dag-driver-3421844605"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 08:30:27.717173 14 driver.go:185] Created execution: id:215 name:"run/813cdf2b-6ff3-4126-9631-6b3b60112417" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125027682 last_update_time_since_epoch:1747125027682
I0513 07:30:01.881256 14 driver.go:252] parent DAG: id:204 name:"run/0590f049-909f-4de3-86a6-74d60f95de6d" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747121327328 last_update_time_since_epoch:1747121327328
I0513 07:30:01.925691 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"204"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","0590f049-909f-4de3-86a6-74d60f95de6d","--dag_execution_id","204","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-f5jll/2025/05/13/pod-log-pipeline-advance-version-f5jll-system-container-driver-960807485"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"204"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","0590f049-909f-4de3-86a6-74d60f95de6d","--dag_execution_id","204","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-f5jll/2025/05/13/pod-log-pipeline-advance-version-f5jll-system-container-driver-960807485"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"204"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","0590f049-909f-4de3-86a6-74d60f95de6d","--dag_execution_id","204","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-f5jll/2025/05/13/pod-log-pipeline-advance-version-f5jll-system-container-driver-960807485"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 07:28:57.415100 15 driver.go:252] parent DAG: id:204 name:"run/0590f049-909f-4de3-86a6-74d60f95de6d" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747121327328 last_update_time_since_epoch:1747121327328 I0513 07:28:57.469712 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 07:28:57.539727 15 driver.go:310] Created execution: id:205 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"Collect Filtered Logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:204}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"Collect Filtered Logs"}} create_time_since_epoch:1747121337507 last_update_time_since_epoch:1747121337507 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","0590f049-909f-4de3-86a6-74d60f95de6d","--execution_id","205","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/0590f049-909f-4de3-86a6-74d60f95de6d/Collect Filtered Logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"0590f049-909f-4de3-86a6-74d60f95de6d\",\"--execution_id\",\"205\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/0590f049-909f-4de3-86a6-74d60f95de6d/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-f5jll/2025/05/13/pod-log-pipeline-advance-version-f5jll-system-container-impl-1582906475"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"0590f049-909f-4de3-86a6-74d60f95de6d\",\"--execution_id\",\"205\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/0590f049-909f-4de3-86a6-74d60f95de6d/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"0590f049-909f-4de3-86a6-74d60f95de6d\",\"--execution_id\",\"205\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/0590f049-909f-4de3-86a6-74d60f95de6d/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-f5jll/2025/05/13/pod-log-pipeline-advance-version-f5jll-system-container-impl-1582906475"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"0590f049-909f-4de3-86a6-74d60f95de6d\",\"--execution_id\",\"205\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/0590f049-909f-4de3-86a6-74d60f95de6d/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"0590f049-909f-4de3-86a6-74d60f95de6d\",\"--execution_id\",\"205\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/0590f049-909f-4de3-86a6-74d60f95de6d/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-f5jll/2025/05/13/pod-log-pipeline-advance-version-f5jll-system-container-impl-1582906475"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"0590f049-909f-4de3-86a6-74d60f95de6d\",\"--execution_id\",\"205\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/0590f049-909f-4de3-86a6-74d60f95de6d/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/0590f049-909f-4de3-86a6-74d60f95de6d/Collect Filtered Logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready",
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"0590f049-909f-4de3-86a6-74d60f95de6d\",\"--execution_id\",\"205\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/0590f049-909f-4de3-86a6-74d60f95de6d/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-f5jll/2025/05/13/pod-log-pipeline-advance-version-f5jll-system-container-impl-1582906475"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"0590f049-909f-4de3-86a6-74d60f95de6d\",\"--execution_id\",\"205\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/0590f049-909f-4de3-86a6-74d60f95de6d/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","0590f049-909f-4de3-86a6-74d60f95de6d","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-f5jll/2025/05/13/pod-log-pipeline-advance-version-f5jll-system-dag-driver-1975691797"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","0590f049-909f-4de3-86a6-74d60f95de6d","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-f5jll/2025/05/13/pod-log-pipeline-advance-version-f5jll-system-dag-driver-1975691797"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"Collect Filtered Logs"}},"upload-to-github":{"cachingOptions":{},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"Upload to GitHub"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","0590f049-909f-4de3-86a6-74d60f95de6d","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"Upload to GitHub\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-f5jll/2025/05/13/pod-log-pipeline-advance-version-f5jll-system-dag-driver-1975691797"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 07:28:47.361506 14 driver.go:185] Created execution: id:204 name:"run/0590f049-909f-4de3-86a6-74d60f95de6d" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747121327328 last_update_time_since_epoch:1747121327328
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"224"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1f730446-7f17-4602-a143-fa3db60067f9","--dag_execution_id","224","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-fjvxx/2025/05/13/pod-log-pipeline-advance-version-fjvxx-system-container-driver-2503112002"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"224"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1f730446-7f17-4602-a143-fa3db60067f9","--dag_execution_id","224","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-fjvxx/2025/05/13/pod-log-pipeline-advance-version-fjvxx-system-container-driver-2503112002"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"224"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1f730446-7f17-4602-a143-fa3db60067f9","--dag_execution_id","224","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-fjvxx/2025/05/13/pod-log-pipeline-advance-version-fjvxx-system-container-driver-2503112002"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 08:41:25.294400 14 driver.go:252] parent DAG: id:224 name:"run/1f730446-7f17-4602-a143-fa3db60067f9" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125675308 last_update_time_since_epoch:1747125675308 I0513 08:41:25.337371 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 08:41:25.412859 14 driver.go:310] Created execution: id:225 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:224}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747125685377 last_update_time_since_epoch:1747125685377 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1f730446-7f17-4602-a143-fa3db60067f9","--execution_id","225","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1f730446-7f17-4602-a143-fa3db60067f9/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
I0513 08:42:36.089287 15 driver.go:252] parent DAG: id:224 name:"run/1f730446-7f17-4602-a143-fa3db60067f9" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125675308 last_update_time_since_epoch:1747125675308
I0513 08:42:36.136885 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1f730446-7f17-4602-a143-fa3db60067f9\",\"--execution_id\",\"225\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1f730446-7f17-4602-a143-fa3db60067f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-fjvxx/2025/05/13/pod-log-pipeline-advance-version-fjvxx-system-container-impl-673697548"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1f730446-7f17-4602-a143-fa3db60067f9\",\"--execution_id\",\"225\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1f730446-7f17-4602-a143-fa3db60067f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1f730446-7f17-4602-a143-fa3db60067f9\",\"--execution_id\",\"225\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1f730446-7f17-4602-a143-fa3db60067f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-fjvxx/2025/05/13/pod-log-pipeline-advance-version-fjvxx-system-container-impl-673697548"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1f730446-7f17-4602-a143-fa3db60067f9\",\"--execution_id\",\"225\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1f730446-7f17-4602-a143-fa3db60067f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1f730446-7f17-4602-a143-fa3db60067f9\",\"--execution_id\",\"225\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1f730446-7f17-4602-a143-fa3db60067f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-fjvxx/2025/05/13/pod-log-pipeline-advance-version-fjvxx-system-container-impl-673697548"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1f730446-7f17-4602-a143-fa3db60067f9\",\"--execution_id\",\"225\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1f730446-7f17-4602-a143-fa3db60067f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1f730446-7f17-4602-a143-fa3db60067f9/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready",
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1f730446-7f17-4602-a143-fa3db60067f9\",\"--execution_id\",\"225\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1f730446-7f17-4602-a143-fa3db60067f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-fjvxx/2025/05/13/pod-log-pipeline-advance-version-fjvxx-system-container-impl-673697548"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1f730446-7f17-4602-a143-fa3db60067f9\",\"--execution_id\",\"225\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1f730446-7f17-4602-a143-fa3db60067f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1f730446-7f17-4602-a143-fa3db60067f9","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-fjvxx/2025/05/13/pod-log-pipeline-advance-version-fjvxx-system-dag-driver-4165243590"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1f730446-7f17-4602-a143-fa3db60067f9","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-fjvxx/2025/05/13/pod-log-pipeline-advance-version-fjvxx-system-dag-driver-4165243590"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1f730446-7f17-4602-a143-fa3db60067f9","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-fjvxx/2025/05/13/pod-log-pipeline-advance-version-fjvxx-system-dag-driver-4165243590"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 08:41:15.338977 14 driver.go:185] Created execution: id:224 name:"run/1f730446-7f17-4602-a143-fa3db60067f9" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125675308 last_update_time_since_epoch:1747125675308
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"188"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","fb714a24-0df2-43cf-8cb2-06486bb9a010","--dag_execution_id","188","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-jfvff/2025/05/13/pod-log-pipeline-advance-version-jfvff-system-container-driver-2568180026"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"188"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","fb714a24-0df2-43cf-8cb2-06486bb9a010","--dag_execution_id","188","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-jfvff/2025/05/13/pod-log-pipeline-advance-version-jfvff-system-container-driver-2568180026"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"188"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","fb714a24-0df2-43cf-8cb2-06486bb9a010","--dag_execution_id","188","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-jfvff/2025/05/13/pod-log-pipeline-advance-version-jfvff-system-container-driver-2568180026"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 06:07:58.668586 15 driver.go:252] parent DAG: id:188 name:"run/fb714a24-0df2-43cf-8cb2-06486bb9a010" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747116468638 last_update_time_since_epoch:1747116468638 I0513 06:07:58.712585 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 06:07:58.801191 15 driver.go:310] Created execution: id:189 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"f7d2838b294b9826fe0e2fb40d5b20cfcb6d695d9cd08eddd83eed493416dc60"}} custom_properties:{key:"cached_execution_id" value:{string_value:"186"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:188}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747116478762 last_update_time_since_epoch:1747116478762 "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
I0513 06:08:08.768946 14 driver.go:252] parent DAG: id:188 name:"run/fb714a24-0df2-43cf-8cb2-06486bb9a010" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747116468638 last_update_time_since_epoch:1747116468638
I0513 06:08:08.815682 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","fb714a24-0df2-43cf-8cb2-06486bb9a010","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-jfvff/2025/05/13/pod-log-pipeline-advance-version-jfvff-system-dag-driver-3230713982"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","fb714a24-0df2-43cf-8cb2-06486bb9a010","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-jfvff/2025/05/13/pod-log-pipeline-advance-version-jfvff-system-dag-driver-3230713982"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","fb714a24-0df2-43cf-8cb2-06486bb9a010","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-jfvff/2025/05/13/pod-log-pipeline-advance-version-jfvff-system-dag-driver-3230713982"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 06:07:48.674184 15 driver.go:185] Created execution: id:188 name:"run/fb714a24-0df2-43cf-8cb2-06486bb9a010" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747116468638 last_update_time_since_epoch:1747116468638
I0513 06:24:53.037471 15 driver.go:252] parent DAG: id:195 name:"run/431825e3-530c-4900-8f9b-8e0742e98294" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747117420625 last_update_time_since_epoch:1747117420625
I0513 06:24:53.091168 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"195"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","431825e3-530c-4900-8f9b-8e0742e98294","--dag_execution_id","195","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-nh6wx/2025/05/13/pod-log-pipeline-advance-version-nh6wx-system-container-driver-363543251"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"195"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","431825e3-530c-4900-8f9b-8e0742e98294","--dag_execution_id","195","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-nh6wx/2025/05/13/pod-log-pipeline-advance-version-nh6wx-system-container-driver-363543251"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"195"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","431825e3-530c-4900-8f9b-8e0742e98294","--dag_execution_id","195","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-nh6wx/2025/05/13/pod-log-pipeline-advance-version-nh6wx-system-container-driver-363543251"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 06:23:50.589335 15 driver.go:252] parent DAG: id:195 name:"run/431825e3-530c-4900-8f9b-8e0742e98294" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747117420625 last_update_time_since_epoch:1747117420625 I0513 06:23:50.633272 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 06:23:50.727973 15 driver.go:310] Created execution: id:196 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"ec7357b8454fa3705a6ec901b80973fd1d4bf2e3f469a60a99bae7e3b4566eda"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:195}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747117430687 last_update_time_since_epoch:1747117430687 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","431825e3-530c-4900-8f9b-8e0742e98294","--execution_id","196","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/431825e3-530c-4900-8f9b-8e0742e98294/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"431825e3-530c-4900-8f9b-8e0742e98294\",\"--execution_id\",\"196\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/431825e3-530c-4900-8f9b-8e0742e98294/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-nh6wx/2025/05/13/pod-log-pipeline-advance-version-nh6wx-system-container-impl-3635549109"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"431825e3-530c-4900-8f9b-8e0742e98294\",\"--execution_id\",\"196\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/431825e3-530c-4900-8f9b-8e0742e98294/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"431825e3-530c-4900-8f9b-8e0742e98294\",\"--execution_id\",\"196\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/431825e3-530c-4900-8f9b-8e0742e98294/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-nh6wx/2025/05/13/pod-log-pipeline-advance-version-nh6wx-system-container-impl-3635549109"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"431825e3-530c-4900-8f9b-8e0742e98294\",\"--execution_id\",\"196\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/431825e3-530c-4900-8f9b-8e0742e98294/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"431825e3-530c-4900-8f9b-8e0742e98294\",\"--execution_id\",\"196\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/431825e3-530c-4900-8f9b-8e0742e98294/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-nh6wx/2025/05/13/pod-log-pipeline-advance-version-nh6wx-system-container-impl-3635549109"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"431825e3-530c-4900-8f9b-8e0742e98294\",\"--execution_id\",\"196\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/431825e3-530c-4900-8f9b-8e0742e98294/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/431825e3-530c-4900-8f9b-8e0742e98294/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready",
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"431825e3-530c-4900-8f9b-8e0742e98294\",\"--execution_id\",\"196\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/431825e3-530c-4900-8f9b-8e0742e98294/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-nh6wx/2025/05/13/pod-log-pipeline-advance-version-nh6wx-system-container-impl-3635549109"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"431825e3-530c-4900-8f9b-8e0742e98294\",\"--execution_id\",\"196\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/431825e3-530c-4900-8f9b-8e0742e98294/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","431825e3-530c-4900-8f9b-8e0742e98294","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-nh6wx/2025/05/13/pod-log-pipeline-advance-version-nh6wx-system-dag-driver-1666669711"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","431825e3-530c-4900-8f9b-8e0742e98294","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-nh6wx/2025/05/13/pod-log-pipeline-advance-version-nh6wx-system-dag-driver-1666669711"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","431825e3-530c-4900-8f9b-8e0742e98294","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-nh6wx/2025/05/13/pod-log-pipeline-advance-version-nh6wx-system-dag-driver-1666669711"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 06:23:40.661100 15 driver.go:185] Created execution: id:195 name:"run/431825e3-530c-4900-8f9b-8e0742e98294" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747117420625 last_update_time_since_epoch:1747117420625
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"218"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","c362aafe-13f2-4560-b80d-d0fa2ed4b952","--dag_execution_id","218","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-p4qjq/2025/05/13/pod-log-pipeline-advance-version-p4qjq-system-container-driver-3390612114"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"218"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","c362aafe-13f2-4560-b80d-d0fa2ed4b952","--dag_execution_id","218","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-p4qjq/2025/05/13/pod-log-pipeline-advance-version-p4qjq-system-container-driver-3390612114"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"218"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","c362aafe-13f2-4560-b80d-d0fa2ed4b952","--dag_execution_id","218","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-p4qjq/2025/05/13/pod-log-pipeline-advance-version-p4qjq-system-container-driver-3390612114"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 08:31:51.342880 15 driver.go:252] parent DAG: id:218 name:"run/c362aafe-13f2-4560-b80d-d0fa2ed4b952" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125101211 last_update_time_since_epoch:1747125101211 I0513 08:31:51.385090 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 08:31:51.456378 15 driver.go:310] Created execution: id:219 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:218}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747125111422 last_update_time_since_epoch:1747125111422 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","c362aafe-13f2-4560-b80d-d0fa2ed4b952","--execution_id","219","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/c362aafe-13f2-4560-b80d-d0fa2ed4b952/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
I0513 08:33:00.983192 14 driver.go:252] parent DAG: id:218 name:"run/c362aafe-13f2-4560-b80d-d0fa2ed4b952" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125101211 last_update_time_since_epoch:1747125101211
I0513 08:33:01.027109 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"c362aafe-13f2-4560-b80d-d0fa2ed4b952\",\"--execution_id\",\"219\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/c362aafe-13f2-4560-b80d-d0fa2ed4b952/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-p4qjq/2025/05/13/pod-log-pipeline-advance-version-p4qjq-system-container-impl-3000831420"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"c362aafe-13f2-4560-b80d-d0fa2ed4b952\",\"--execution_id\",\"219\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/c362aafe-13f2-4560-b80d-d0fa2ed4b952/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"c362aafe-13f2-4560-b80d-d0fa2ed4b952\",\"--execution_id\",\"219\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/c362aafe-13f2-4560-b80d-d0fa2ed4b952/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-p4qjq/2025/05/13/pod-log-pipeline-advance-version-p4qjq-system-container-impl-3000831420"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"c362aafe-13f2-4560-b80d-d0fa2ed4b952\",\"--execution_id\",\"219\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/c362aafe-13f2-4560-b80d-d0fa2ed4b952/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"c362aafe-13f2-4560-b80d-d0fa2ed4b952\",\"--execution_id\",\"219\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/c362aafe-13f2-4560-b80d-d0fa2ed4b952/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-p4qjq/2025/05/13/pod-log-pipeline-advance-version-p4qjq-system-container-impl-3000831420"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"c362aafe-13f2-4560-b80d-d0fa2ed4b952\",\"--execution_id\",\"219\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/c362aafe-13f2-4560-b80d-d0fa2ed4b952/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/c362aafe-13f2-4560-b80d-d0fa2ed4b952/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready",
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"c362aafe-13f2-4560-b80d-d0fa2ed4b952\",\"--execution_id\",\"219\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/c362aafe-13f2-4560-b80d-d0fa2ed4b952/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-p4qjq/2025/05/13/pod-log-pipeline-advance-version-p4qjq-system-container-impl-3000831420"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"c362aafe-13f2-4560-b80d-d0fa2ed4b952\",\"--execution_id\",\"219\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/c362aafe-13f2-4560-b80d-d0fa2ed4b952/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","c362aafe-13f2-4560-b80d-d0fa2ed4b952","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-p4qjq/2025/05/13/pod-log-pipeline-advance-version-p4qjq-system-dag-driver-2518136982"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","c362aafe-13f2-4560-b80d-d0fa2ed4b952","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-p4qjq/2025/05/13/pod-log-pipeline-advance-version-p4qjq-system-dag-driver-2518136982"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","c362aafe-13f2-4560-b80d-d0fa2ed4b952","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-p4qjq/2025/05/13/pod-log-pipeline-advance-version-p4qjq-system-dag-driver-2518136982"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 08:31:41.241898 16 driver.go:185] Created execution: id:218 name:"run/c362aafe-13f2-4560-b80d-d0fa2ed4b952" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125101211 last_update_time_since_epoch:1747125101211
I0514 00:22:16.723376 14 driver.go:252] parent DAG: id:229 name:"run/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747181999421 last_update_time_since_epoch:1747181999421
I0514 00:22:16.770723 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"229"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","cf5cfd83-7f07-4398-9bc4-7e2b52d92a24","--dag_execution_id","229","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-driver-3520029909"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"229"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","cf5cfd83-7f07-4398-9bc4-7e2b52d92a24","--dag_execution_id","229","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-driver-3520029909"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash,error,fail,oomkilled,evicted","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\n h2 {{ color: #34495e; margin-top: 40px; }}\n h3 {{ margin-top: 20px; color: #2e86de; }}\n pre {{\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }}\n section {{ margin-bottom: 40px; }}\n .highlight {{\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"229"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","cf5cfd83-7f07-4398-9bc4-7e2b52d92a24","--dag_execution_id","229","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-driver-3520029909"}}}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "defaultValue": "crash,error,fail,oomkilled,evicted", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\n h2 {{ color: #34495e; margin-top: 40px; }}\n h3 {{ margin-top: 20px; color: #2e86de; }}\n pre {{\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }}\n section {{ margin-bottom: 40px; }}\n .highlight {{\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0514 00:20:09.494521 15 driver.go:252] parent DAG: id:229 name:"run/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747181999421 last_update_time_since_epoch:1747181999421 I0514 00:20:09.537018 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0514 00:20:09.606587 15 driver.go:310] Created execution: id:230 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:229}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747182009574 last_update_time_since_epoch:1747182009574 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","cf5cfd83-7f07-4398-9bc4-7e2b52d92a24","--execution_id","230","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"crash,error,fail,oomkilled,evicted\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\n h2 {{ color: #34495e; margin-top: 40px; }}\n h3 {{ margin-top: 20px; color: #2e86de; }}\n pre {{\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }}\n section {{ margin-bottom: 40px; }}\n .highlight {{\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--execution_id\",\"230\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-impl-2405826195"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--execution_id\",\"230\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--execution_id\",\"230\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-impl-2405826195"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--execution_id\",\"230\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--execution_id\",\"230\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-impl-2405826195"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--execution_id\",\"230\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"crash,error,fail,oomkilled,evicted", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "crash,error,fail,oomkilled,evicted",
event_keywords: str = "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled"):
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--execution_id\",\"230\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-impl-2405826195"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--execution_id\",\"230\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "defaultValue": "crash,error,fail,oomkilled,evicted", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","cf5cfd83-7f07-4398-9bc4-7e2b52d92a24","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-dag-driver-2332657773"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","cf5cfd83-7f07-4398-9bc4-7e2b52d92a24","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-dag-driver-2332657773"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","cf5cfd83-7f07-4398-9bc4-7e2b52d92a24","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-dag-driver-2332657773"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0514 00:19:59.453384 16 driver.go:185] Created execution: id:229 name:"run/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747181999421 last_update_time_since_epoch:1747181999421
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"221"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1c14d7d4-92f0-477c-83ad-b75bb4fa0051","--dag_execution_id","221","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-sspss/2025/05/13/pod-log-pipeline-advance-version-sspss-system-container-driver-4198297916"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"221"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1c14d7d4-92f0-477c-83ad-b75bb4fa0051","--dag_execution_id","221","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-sspss/2025/05/13/pod-log-pipeline-advance-version-sspss-system-container-driver-4198297916"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"221"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1c14d7d4-92f0-477c-83ad-b75bb4fa0051","--dag_execution_id","221","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-sspss/2025/05/13/pod-log-pipeline-advance-version-sspss-system-container-driver-4198297916"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 08:39:23.124133 13 driver.go:252] parent DAG: id:221 name:"run/1c14d7d4-92f0-477c-83ad-b75bb4fa0051" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125553097 last_update_time_since_epoch:1747125553097 I0513 08:39:23.177231 13 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 08:39:23.255520 13 driver.go:310] Created execution: id:222 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:221}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747125563219 last_update_time_since_epoch:1747125563219 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1c14d7d4-92f0-477c-83ad-b75bb4fa0051","--execution_id","222","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1c14d7d4-92f0-477c-83ad-b75bb4fa0051/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
I0513 08:40:33.875115 13 driver.go:252] parent DAG: id:221 name:"run/1c14d7d4-92f0-477c-83ad-b75bb4fa0051" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125553097 last_update_time_since_epoch:1747125553097
I0513 08:40:33.918577 13 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1c14d7d4-92f0-477c-83ad-b75bb4fa0051\",\"--execution_id\",\"222\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1c14d7d4-92f0-477c-83ad-b75bb4fa0051/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-sspss/2025/05/13/pod-log-pipeline-advance-version-sspss-system-container-impl-1726354290"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1c14d7d4-92f0-477c-83ad-b75bb4fa0051\",\"--execution_id\",\"222\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1c14d7d4-92f0-477c-83ad-b75bb4fa0051/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1c14d7d4-92f0-477c-83ad-b75bb4fa0051\",\"--execution_id\",\"222\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1c14d7d4-92f0-477c-83ad-b75bb4fa0051/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-sspss/2025/05/13/pod-log-pipeline-advance-version-sspss-system-container-impl-1726354290"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1c14d7d4-92f0-477c-83ad-b75bb4fa0051\",\"--execution_id\",\"222\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1c14d7d4-92f0-477c-83ad-b75bb4fa0051/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1c14d7d4-92f0-477c-83ad-b75bb4fa0051\",\"--execution_id\",\"222\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1c14d7d4-92f0-477c-83ad-b75bb4fa0051/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-sspss/2025/05/13/pod-log-pipeline-advance-version-sspss-system-container-impl-1726354290"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1c14d7d4-92f0-477c-83ad-b75bb4fa0051\",\"--execution_id\",\"222\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1c14d7d4-92f0-477c-83ad-b75bb4fa0051/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1c14d7d4-92f0-477c-83ad-b75bb4fa0051/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready",
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1c14d7d4-92f0-477c-83ad-b75bb4fa0051\",\"--execution_id\",\"222\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1c14d7d4-92f0-477c-83ad-b75bb4fa0051/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-sspss/2025/05/13/pod-log-pipeline-advance-version-sspss-system-container-impl-1726354290"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1c14d7d4-92f0-477c-83ad-b75bb4fa0051\",\"--execution_id\",\"222\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1c14d7d4-92f0-477c-83ad-b75bb4fa0051/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1c14d7d4-92f0-477c-83ad-b75bb4fa0051","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-sspss/2025/05/13/pod-log-pipeline-advance-version-sspss-system-dag-driver-2424267844"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1c14d7d4-92f0-477c-83ad-b75bb4fa0051","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-sspss/2025/05/13/pod-log-pipeline-advance-version-sspss-system-dag-driver-2424267844"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1c14d7d4-92f0-477c-83ad-b75bb4fa0051","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-sspss/2025/05/13/pod-log-pipeline-advance-version-sspss-system-dag-driver-2424267844"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 08:39:13.160563 14 driver.go:185] Created execution: id:221 name:"run/1c14d7d4-92f0-477c-83ad-b75bb4fa0051" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747125553097 last_update_time_since_epoch:1747125553097
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"201"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","71b763ed-0c79-4097-9f90-6a2500a3e6d1","--dag_execution_id","201","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-tcgsg/2025/05/13/pod-log-pipeline-advance-version-tcgsg-system-container-driver-1366828380"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"201"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","71b763ed-0c79-4097-9f90-6a2500a3e6d1","--dag_execution_id","201","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-tcgsg/2025/05/13/pod-log-pipeline-advance-version-tcgsg-system-container-driver-1366828380"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"201"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","71b763ed-0c79-4097-9f90-6a2500a3e6d1","--dag_execution_id","201","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-tcgsg/2025/05/13/pod-log-pipeline-advance-version-tcgsg-system-container-driver-1366828380"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 07:02:32.969618 15 driver.go:252] parent DAG: id:201 name:"run/71b763ed-0c79-4097-9f90-6a2500a3e6d1" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747119743005 last_update_time_since_epoch:1747119743005 I0513 07:02:33.009525 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 07:02:33.103573 15 driver.go:310] Created execution: id:202 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"cache_fingerprint" value:{string_value:"ec7357b8454fa3705a6ec901b80973fd1d4bf2e3f469a60a99bae7e3b4566eda"}} custom_properties:{key:"cached_execution_id" value:{string_value:"196"}} custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:201}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747119753067 last_update_time_since_epoch:1747119753067 "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
I0513 07:02:43.070210 14 driver.go:252] parent DAG: id:201 name:"run/71b763ed-0c79-4097-9f90-6a2500a3e6d1" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747119743005 last_update_time_since_epoch:1747119743005
I0513 07:02:43.122833 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","71b763ed-0c79-4097-9f90-6a2500a3e6d1","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-tcgsg/2025/05/13/pod-log-pipeline-advance-version-tcgsg-system-dag-driver-577476708"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","71b763ed-0c79-4097-9f90-6a2500a3e6d1","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-tcgsg/2025/05/13/pod-log-pipeline-advance-version-tcgsg-system-dag-driver-577476708"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","71b763ed-0c79-4097-9f90-6a2500a3e6d1","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-tcgsg/2025/05/13/pod-log-pipeline-advance-version-tcgsg-system-dag-driver-577476708"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 07:02:23.036897 14 driver.go:185] Created execution: id:201 name:"run/71b763ed-0c79-4097-9f90-6a2500a3e6d1" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747119743005 last_update_time_since_epoch:1747119743005
I0513 08:04:00.710185 14 driver.go:252] parent DAG: id:211 name:"run/1e6b76a6-d927-435b-abe2-596f09f41415" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747123364024 last_update_time_since_epoch:1747123364024
I0513 08:04:00.752116 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"211"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1e6b76a6-d927-435b-abe2-596f09f41415","--dag_execution_id","211","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-twg4s/2025/05/13/pod-log-pipeline-advance-version-twg4s-system-container-driver-4166101455"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"211"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1e6b76a6-d927-435b-abe2-596f09f41415","--dag_execution_id","211","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-twg4s/2025/05/13/pod-log-pipeline-advance-version-twg4s-system-container-driver-4166101455"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"211"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1e6b76a6-d927-435b-abe2-596f09f41415","--dag_execution_id","211","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-twg4s/2025/05/13/pod-log-pipeline-advance-version-twg4s-system-container-driver-4166101455"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 08:02:54.113877 14 driver.go:252] parent DAG: id:211 name:"run/1e6b76a6-d927-435b-abe2-596f09f41415" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747123364024 last_update_time_since_epoch:1747123364024 I0513 08:02:54.158152 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 08:02:54.233198 14 driver.go:310] Created execution: id:212 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"Collect Filtered Logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:211}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"Collect Filtered Logs"}} create_time_since_epoch:1747123374198 last_update_time_since_epoch:1747123374198 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1e6b76a6-d927-435b-abe2-596f09f41415","--execution_id","212","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1e6b76a6-d927-435b-abe2-596f09f41415/Collect Filtered Logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\n event_keywords: str = \"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\"):\n\n import subprocess\n import os\n import html\n import re\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n log_kw_list = [kw.strip() for kw in log_keywords.split(\",\")]\n event_kw_list = [kw.strip() for kw in event_keywords.split(\",\")]\n\n def highlight_keywords(text: str, keywords: list[str]) -\u003e tuple[str, list[str]]:\n found = set()\n for kw in keywords:\n if kw in text:\n found.add(kw)\n text = re.sub(f\"({re.escape(kw)})\", r\"\u003cspan style='color:red; font-weight:bold;'\u003e\\1\u003c/span\u003e\", text, flags=re.IGNORECASE)\n return text, sorted(list(found))\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: kubectl get pods 결과\u003c/h2\u003e\u003cpre\u003e{pods_output}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(f\"\u003csection\u003e\u003ch2\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\u003c/h2\u003e\u003c/section\u003e\")\n\n for line in pod_lines:\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace} --tail=100\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = \"\\n\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs = \"\\n\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\n\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\n all_found = list(set(describe_found + log_found))\n\n if matched_describe or matched_logs:\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: \u003ccode\u003e{pod_name}\u003c/code\u003e (Namespace: \u003ccode\u003e{namespace}\u003c/code\u003e)\u003c/h2\u003e\\n\")\n if all_found:\n f.write(f\"\u003cp\u003e⚠️ 감지된 키워드: \u003cstrong\u003e{', '.join(all_found)}\u003c/strong\u003e\u003c/p\u003e\")\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{matched_describe}\u003c/pre\u003e\\n\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (마지막 10줄)\u003c/h3\u003e\u003cpre\u003e{matched_logs}\u003c/pre\u003e\\n\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\n namespace_events[namespace] = highlighted_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eNamespace: \u003ccode\u003e{ns}\u003c/code\u003e - Events\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1e6b76a6-d927-435b-abe2-596f09f41415\",\"--execution_id\",\"212\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1e6b76a6-d927-435b-abe2-596f09f41415/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-twg4s/2025/05/13/pod-log-pipeline-advance-version-twg4s-system-container-impl-2126002369"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1e6b76a6-d927-435b-abe2-596f09f41415\",\"--execution_id\",\"212\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1e6b76a6-d927-435b-abe2-596f09f41415/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1e6b76a6-d927-435b-abe2-596f09f41415\",\"--execution_id\",\"212\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1e6b76a6-d927-435b-abe2-596f09f41415/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-twg4s/2025/05/13/pod-log-pipeline-advance-version-twg4s-system-container-impl-2126002369"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1e6b76a6-d927-435b-abe2-596f09f41415\",\"--execution_id\",\"212\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1e6b76a6-d927-435b-abe2-596f09f41415/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1e6b76a6-d927-435b-abe2-596f09f41415\",\"--execution_id\",\"212\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1e6b76a6-d927-435b-abe2-596f09f41415/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-twg4s/2025/05/13/pod-log-pipeline-advance-version-twg4s-system-container-impl-2126002369"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1e6b76a6-d927-435b-abe2-596f09f41415\",\"--execution_id\",\"212\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1e6b76a6-d927-435b-abe2-596f09f41415/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1e6b76a6-d927-435b-abe2-596f09f41415/Collect Filtered Logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready",
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1e6b76a6-d927-435b-abe2-596f09f41415\",\"--execution_id\",\"212\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1e6b76a6-d927-435b-abe2-596f09f41415/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-twg4s/2025/05/13/pod-log-pipeline-advance-version-twg4s-system-container-impl-2126002369"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"1e6b76a6-d927-435b-abe2-596f09f41415\",\"--execution_id\",\"212\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/1e6b76a6-d927-435b-abe2-596f09f41415/Collect Filtered Logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\n event_keywords: str = \\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\"):\\n\\n import subprocess\\n import os\\n import html\\n import re\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n log_kw_list = [kw.strip() for kw in log_keywords.split(\\\",\\\")]\\n event_kw_list = [kw.strip() for kw in event_keywords.split(\\\",\\\")]\\n\\n def highlight_keywords(text: str, keywords: list[str]) -\\u003e tuple[str, list[str]]:\\n found = set()\\n for kw in keywords:\\n if kw in text:\\n found.add(kw)\\n text = re.sub(f\\\"({re.escape(kw)})\\\", r\\\"\\u003cspan style='color:red; font-weight:bold;'\\u003e\\\\1\\u003c/span\\u003e\\\", text, flags=re.IGNORECASE)\\n return text, sorted(list(found))\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: kubectl get pods 결과\\u003c/h2\\u003e\\u003cpre\\u003e{pods_output}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eDEBUG: 총 검사 대상 Pod 수: {len(pod_lines)}\\u003c/h2\\u003e\\u003c/section\\u003e\\\")\\n\\n for line in pod_lines:\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace} --tail=100\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = \\\"\\\\n\\\".join([l for l in logs.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw.strip().splitlines()[-10:]) # 마지막 10줄\\n\\n matched_describe, describe_found = highlight_keywords(html.escape(matched_describe), log_kw_list)\\n matched_logs, log_found = highlight_keywords(html.escape(matched_logs), log_kw_list)\\n all_found = list(set(describe_found + log_found))\\n\\n if matched_describe or matched_logs:\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: \\u003ccode\\u003e{pod_name}\\u003c/code\\u003e (Namespace: \\u003ccode\\u003e{namespace}\\u003c/code\\u003e)\\u003c/h2\\u003e\\\\n\\\")\\n if all_found:\\n f.write(f\\\"\\u003cp\\u003e⚠️ 감지된 키워드: \\u003cstrong\\u003e{', '.join(all_found)}\\u003c/strong\\u003e\\u003c/p\\u003e\\\")\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{matched_describe}\\u003c/pre\\u003e\\\\n\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (마지막 10줄)\\u003c/h3\\u003e\\u003cpre\\u003e{matched_logs}\\u003c/pre\\u003e\\\\n\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n highlighted_events, _ = highlight_keywords(html.escape(matched_events), event_kw_list)\\n namespace_events[namespace] = highlighted_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eNamespace: \\u003ccode\\u003e{ns}\\u003c/code\\u003e - Events\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1e6b76a6-d927-435b-abe2-596f09f41415","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-twg4s/2025/05/13/pod-log-pipeline-advance-version-twg4s-system-dag-driver-1446001731"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1e6b76a6-d927-435b-abe2-596f09f41415","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-twg4s/2025/05/13/pod-log-pipeline-advance-version-twg4s-system-dag-driver-1446001731"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"Collect Filtered Logs"}},"upload-to-github":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","1e6b76a6-d927-435b-abe2-596f09f41415","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"Collect Filtered Logs\"}},\"upload-to-github\":{\"cachingOptions\":{\"enableCache\":true},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-twg4s/2025/05/13/pod-log-pipeline-advance-version-twg4s-system-dag-driver-1446001731"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 08:02:44.054761 14 driver.go:185] Created execution: id:211 name:"run/1e6b76a6-d927-435b-abe2-596f09f41415" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747123364024 last_update_time_since_epoch:1747123364024
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"235"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","8daa4a6b-1876-4070-882c-ccc8e2a43d5d","--dag_execution_id","235","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-container-driver-649163649"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"235"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","8daa4a6b-1876-4070-882c-ccc8e2a43d5d","--dag_execution_id","235","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-container-driver-649163649"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash,error,fail,oomkilled,evicted","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\n h2 {{ color: #34495e; margin-top: 40px; }}\n h3 {{ margin-top: 20px; color: #2e86de; }}\n pre {{\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }}\n section {{ margin-bottom: 40px; }}\n .highlight {{\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"235"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","8daa4a6b-1876-4070-882c-ccc8e2a43d5d","--dag_execution_id","235","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-container-driver-649163649"}}}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "defaultValue": "crash,error,fail,oomkilled,evicted", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\n h2 {{ color: #34495e; margin-top: 40px; }}\n h3 {{ margin-top: 20px; color: #2e86de; }}\n pre {{\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }}\n section {{ margin-bottom: 40px; }}\n .highlight {{\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0514 00:26:23.290304 14 driver.go:252] parent DAG: id:235 name:"run/8daa4a6b-1876-4070-882c-ccc8e2a43d5d" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747182373265 last_update_time_since_epoch:1747182373265 I0514 00:26:23.335682 14 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0514 00:26:23.418751 14 driver.go:310] Created execution: id:236 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:235}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747182383380 last_update_time_since_epoch:1747182383380 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","8daa4a6b-1876-4070-882c-ccc8e2a43d5d","--execution_id","236","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"crash,error,fail,oomkilled,evicted\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\n h2 {{ color: #34495e; margin-top: 40px; }}\n h3 {{ margin-top: 20px; color: #2e86de; }}\n pre {{\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }}\n section {{ margin-bottom: 40px; }}\n .highlight {{\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }}\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--execution_id\",\"236\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-container-impl-1664075631"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--execution_id\",\"236\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--execution_id\",\"236\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-container-impl-1664075631"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--execution_id\",\"236\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--execution_id\",\"236\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-container-impl-1664075631"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--execution_id\",\"236\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"crash,error,fail,oomkilled,evicted", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "crash,error,fail,oomkilled,evicted",
event_keywords: str = "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled"):
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--execution_id\",\"236\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-container-impl-1664075631"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--execution_id\",\"236\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\n h2 {{ color: #34495e; margin-top: 40px; }}\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\n pre {{\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }}\\n section {{ margin-bottom: 40px; }}\\n .highlight {{\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }}\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "defaultValue": "crash,error,fail,oomkilled,evicted", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","8daa4a6b-1876-4070-882c-ccc8e2a43d5d","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-dag-driver-4218789953"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","8daa4a6b-1876-4070-882c-ccc8e2a43d5d","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-dag-driver-4218789953"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","8daa4a6b-1876-4070-882c-ccc8e2a43d5d","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-dag-driver-4218789953"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0514 00:26:13.298634 15 driver.go:185] Created execution: id:235 name:"run/8daa4a6b-1876-4070-882c-ccc8e2a43d5d" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747182373265 last_update_time_since_epoch:1747182373265
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"227"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","d0e6f244-ec8b-46a8-ba5c-18eebf6904f9","--dag_execution_id","227","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-zdlvz/2025/05/13/pod-log-pipeline-advance-version-zdlvz-system-container-driver-4129444982"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"227"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","d0e6f244-ec8b-46a8-ba5c-18eebf6904f9","--dag_execution_id","227","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-zdlvz/2025/05/13/pod-log-pipeline-advance-version-zdlvz-system-container-driver-4129444982"}}}
{"executorLabel":"exec-collect-filtered-logs","inputDefinitions":{"parameters":{"event_keywords":{"defaultValue":"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"crash,error,fail,oomkilled,evicted","isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\n h2 { color: #34495e; margin-top: 40px; }\n h3 { margin-top: 20px; color: #2e86de; }\n pre {\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }\n section { margin-bottom: 40px; }\n .highlight {\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"227"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-pipeline-advance-version","--run_id","d0e6f244-ec8b-46a8-ba5c-18eebf6904f9","--dag_execution_id","227","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"crash,error,fail,oomkilled,evicted\",\"isOptional\":true,\"parameterType\":\"STRING\"}}},\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-zdlvz/2025/05/13/pod-log-pipeline-advance-version-zdlvz-system-container-driver-4129444982"}}}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "defaultValue": "crash,error,fail,oomkilled,evicted", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\n h2 { color: #34495e; margin-top: 40px; }\n h3 { margin-top: 20px; color: #2e86de; }\n pre {\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }\n section { margin-bottom: 40px; }\n .highlight {\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n" I0513 08:58:45.277509 15 driver.go:252] parent DAG: id:227 name:"run/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747126716346 last_update_time_since_epoch:1747126716346 I0513 08:58:45.320847 15 driver.go:926] parent DAG input parameters: map[branch:string_value:"report" event_keywords:string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory" github_token:string_value:"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" log_keywords:string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready" repo_url:string_value:"https://github.com/Rosa1026/Kubeflow.git"], artifacts: map[] I0513 08:58:45.397893 15 driver.go:310] Created execution: id:228 type_id:14 type:"system.ContainerExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:"collect-filtered-logs"}} custom_properties:{key:"image" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}}}} custom_properties:{key:"namespace" value:{string_value:""}} custom_properties:{key:"parent_dag_id" value:{int_value:227}} custom_properties:{key:"pod_name" value:{string_value:""}} custom_properties:{key:"pod_uid" value:{string_value:""}} custom_properties:{key:"task_name" value:{string_value:"collect-filtered-logs"}} create_time_since_epoch:1747126725361 last_update_time_since_epoch:1747126725361 {"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-pipeline-advance-version","--run_id","d0e6f244-ec8b-46a8-ba5c-18eebf6904f9","--execution_id","228","--executor_input","{\"inputs\":{\"parameterValues\":{\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\", \"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\"}}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"inputDefinitions\":{\"parameters\":{\"event_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\", \"isOptional\":true}, \"log_keywords\":{\"parameterType\":\"STRING\", \"defaultValue\":\"crash,error,fail,oomkilled,evicted\", \"isOptional\":true}}}, \"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset],\n log_keywords: str = \"crash,error,fail,oomkilled,evicted\",\n event_keywords: str = \"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\"):\n\n import os\n import subprocess\n from datetime import datetime\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n log_kw_list = [k.strip() for k in log_keywords.split(\",\") if k.strip()]\n event_kw_list = [k.strip() for k in event_keywords.split(\",\") if k.strip()]\n\n def highlight_keywords(text, keywords):\n for kw in keywords:\n text = text.replace(kw, f\"\u003cspan class='highlight'\u003e{kw}\u003c/span\u003e\")\n return text\n\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n pods_output = subprocess.getoutput(\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\")\n pod_lines = pods_output.strip().split(\"\\n\")\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\n h2 { color: #34495e; margin-top: 40px; }\n h3 { margin-top: 20px; color: #2e86de; }\n pre {\n background-color: #f0f4f8;\n padding: 12px;\n border-left: 5px solid #3498db;\n overflow-x: auto;\n white-space: pre-wrap;\n }\n section { margin-bottom: 40px; }\n .highlight {\n color: red;\n font-weight: bold;\n background-color: #ffe6e6;\n padding: 2px 4px;\n border-radius: 4px;\n }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \u003cp\u003eGenerated at: {}\u003c/p\u003e\n \"\"\".format(datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name = line.split()\n except ValueError:\n continue\n\n print(f\"Checking pod: {pod_name} in namespace: {namespace}\")\n describe = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n logs = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = \"\\n\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\n matched_logs = \"\\n\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\n\n if matched_describe or matched_logs:\n found_keywords = set()\n for kw in log_kw_list:\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\n found_keywords.add(kw)\n\n keyword_display = \", \".join(sorted(found_keywords))\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace})\u003cbr\u003e\u003csmall\u003eDetected keywords: {keyword_display}\u003c/small\u003e\u003c/h2\u003e\")\n\n if matched_describe:\n f.write(f\"\u003ch3\u003eDescribe\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_describe, log_kw_list)}\u003c/pre\u003e\")\n if matched_logs:\n f.write(f\"\u003ch3\u003eLogs (latest 10 matches)\u003c/h3\u003e\u003cpre\u003e{highlight_keywords(matched_logs, log_kw_list)}\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = \"\\n\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{highlight_keywords(events, event_kw_list)}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"d0e6f244-ec8b-46a8-ba5c-18eebf6904f9\",\"--execution_id\",\"228\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-zdlvz/2025/05/13/pod-log-pipeline-advance-version-zdlvz-system-container-impl-2160945376"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"d0e6f244-ec8b-46a8-ba5c-18eebf6904f9\",\"--execution_id\",\"228\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"d0e6f244-ec8b-46a8-ba5c-18eebf6904f9\",\"--execution_id\",\"228\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-zdlvz/2025/05/13/pod-log-pipeline-advance-version-zdlvz-system-container-impl-2160945376"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"d0e6f244-ec8b-46a8-ba5c-18eebf6904f9\",\"--execution_id\",\"228\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"d0e6f244-ec8b-46a8-ba5c-18eebf6904f9\",\"--execution_id\",\"228\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-zdlvz/2025/05/13/pod-log-pipeline-advance-version-zdlvz-system-container-impl-2160945376"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"d0e6f244-ec8b-46a8-ba5c-18eebf6904f9\",\"--execution_id\",\"228\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
{"inputs":{"parameterValues":{"event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory", "log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}}, "outputs":{"artifacts":{"output_report":{"artifacts":[{"type":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}, "uri":"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9/collect-filtered-logs/output_report"}]}}, "outputFile":"/tmp/kfp_outputs/output_metadata.json"}}
{"inputDefinitions":{"parameters":{"event_keywords":{"parameterType":"STRING", "defaultValue":"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "isOptional":true}, "log_keywords":{"parameterType":"STRING", "defaultValue":"crash,error,fail,oomkilled,evicted", "isOptional":true}}}, "outputDefinitions":{"artifacts":{"output_report":{"artifactType":{"schemaTitle":"system.Dataset", "schemaVersion":"0.0.1"}}}}, "executorLabel":"exec-collect-filtered-logs"}
log_keywords: str = "crash,error,fail,oomkilled,evicted",
event_keywords: str = "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled"):
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"d0e6f244-ec8b-46a8-ba5c-18eebf6904f9\",\"--execution_id\",\"228\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-zdlvz/2025/05/13/pod-log-pipeline-advance-version-zdlvz-system-container-impl-2160945376"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"d0e6f244-ec8b-46a8-ba5c-18eebf6904f9\",\"--execution_id\",\"228\",\"--executor_input\",\"{\\\"inputs\\\":{\\\"parameterValues\\\":{\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\", \\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\"}}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\", \\\"isOptional\\\":true}, \\\"log_keywords\\\":{\\\"parameterType\\\":\\\"STRING\\\", \\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\", \\\"isOptional\\\":true}}}, \\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset],\\n log_keywords: str = \\\"crash,error,fail,oomkilled,evicted\\\",\\n event_keywords: str = \\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\"):\\n\\n import os\\n import subprocess\\n from datetime import datetime\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\",\\\") if k.strip()]\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\",\\\") if k.strip()]\\n\\n def highlight_keywords(text, keywords):\\n for kw in keywords:\\n text = text.replace(kw, f\\\"\\u003cspan class='highlight'\\u003e{kw}\\u003c/span\\u003e\\\")\\n return text\\n\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n pods_output = subprocess.getoutput(\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\")\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }\\n h1 { color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }\\n h2 { color: #34495e; margin-top: 40px; }\\n h3 { margin-top: 20px; color: #2e86de; }\\n pre {\\n background-color: #f0f4f8;\\n padding: 12px;\\n border-left: 5px solid #3498db;\\n overflow-x: auto;\\n white-space: pre-wrap;\\n }\\n section { margin-bottom: 40px; }\\n .highlight {\\n color: red;\\n font-weight: bold;\\n background-color: #ffe6e6;\\n padding: 2px 4px;\\n border-radius: 4px;\\n }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\u003cp\\u003eGenerated at: {}\\u003c/p\\u003e\\n \\\"\\\"\\\".format(datetime.now().strftime(\\\"%Y-%m-%d %H:%M:%S\\\")))\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name = line.split()\\n except ValueError:\\n continue\\n\\n print(f\\\"Checking pod: {pod_name} in namespace: {namespace}\\\")\\n describe = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n logs = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = \\\"\\\\n\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\n matched_logs = \\\"\\\\n\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\n\\n if matched_describe or matched_logs:\\n found_keywords = set()\\n for kw in log_kw_list:\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\n found_keywords.add(kw)\\n\\n keyword_display = \\\", \\\".join(sorted(found_keywords))\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace})\\u003cbr\\u003e\\u003csmall\\u003eDetected keywords: {keyword_display}\\u003c/small\\u003e\\u003c/h2\\u003e\\\")\\n\\n if matched_describe:\\n f.write(f\\\"\\u003ch3\\u003eDescribe\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(f\\\"\\u003ch3\\u003eLogs (latest 10 matches)\\u003c/h3\\u003e\\u003cpre\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = \\\"\\\\n\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{highlight_keywords(events, event_kw_list)}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}"defaultValue": "CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled", "defaultValue": "crash,error,fail,oomkilled,evicted", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","d0e6f244-ec8b-46a8-ba5c-18eebf6904f9","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-zdlvz/2025/05/13/pod-log-pipeline-advance-version-zdlvz-system-dag-driver-1383398674"}}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","d0e6f244-ec8b-46a8-ba5c-18eebf6904f9","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-zdlvz/2025/05/13/pod-log-pipeline-advance-version-zdlvz-system-dag-driver-1383398674"}}}
{"dag":{"tasks":{"collect-filtered-logs":{"cachingOptions":{},"componentRef":{"name":"comp-collect-filtered-logs"},"inputs":{"parameters":{"event_keywords":{"componentInputParameter":"event_keywords"},"log_keywords":{"componentInputParameter":"log_keywords"}}},"taskInfo":{"name":"collect-filtered-logs"}},"upload-to-github":{"cachingOptions":{},"componentRef":{"name":"comp-upload-to-github"},"dependentTasks":["collect-filtered-logs"],"inputs":{"artifacts":{"report_file":{"taskOutputArtifact":{"outputArtifactKey":"output_report","producerTask":"collect-filtered-logs"}}},"parameters":{"branch":{"componentInputParameter":"branch"},"github_token":{"componentInputParameter":"github_token"},"repo_url":{"componentInputParameter":"repo_url"}}},"taskInfo":{"name":"upload-to-github"}}}},"inputDefinitions":{"parameters":{"branch":{"defaultValue":"report","isOptional":true,"parameterType":"STRING"},"event_keywords":{"defaultValue":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","isOptional":true,"parameterType":"STRING"},"github_token":{"defaultValue":"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","isOptional":true,"parameterType":"STRING"},"log_keywords":{"defaultValue":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","isOptional":true,"parameterType":"STRING"},"repo_url":{"defaultValue":"https://github.com/Rosa1026/Kubeflow.git","isOptional":true,"parameterType":"STRING"}}}}
{"parameterValues":{"branch":"report","event_keywords":"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory","github_token":"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","log_keywords":"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready","repo_url":"https://github.com/Rosa1026/Kubeflow.git"}}
ARGO_TEMPLATE: {"name":"system-dag-driver","inputs":{"parameters":[{"name":"component","value":"{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}"},{"name":"runtime-config","default":"","value":"{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}"},{"name":"task","default":"","value":""},{"name":"parent-dag-id","default":"0","value":"0"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"driver-type","default":"DAG","value":"ROOT_DAG"}]},"outputs":{"parameters":[{"name":"execution-id","valueFrom":{"path":"/tmp/outputs/execution-id"}},{"name":"iteration-count","valueFrom":{"path":"/tmp/outputs/iteration-count","default":"0"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","ROOT_DAG","--pipeline_name","pod-log-pipeline-advance-version","--run_id","d0e6f244-ec8b-46a8-ba5c-18eebf6904f9","--dag_execution_id","0","--component","{\"dag\":{\"tasks\":{\"collect-filtered-logs\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"inputs\":{\"parameters\":{\"event_keywords\":{\"componentInputParameter\":\"event_keywords\"},\"log_keywords\":{\"componentInputParameter\":\"log_keywords\"}}},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}},\"upload-to-github\":{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-upload-to-github\"},\"dependentTasks\":[\"collect-filtered-logs\"],\"inputs\":{\"artifacts\":{\"report_file\":{\"taskOutputArtifact\":{\"outputArtifactKey\":\"output_report\",\"producerTask\":\"collect-filtered-logs\"}}},\"parameters\":{\"branch\":{\"componentInputParameter\":\"branch\"},\"github_token\":{\"componentInputParameter\":\"github_token\"},\"repo_url\":{\"componentInputParameter\":\"repo_url\"}}},\"taskInfo\":{\"name\":\"upload-to-github\"}}}},\"inputDefinitions\":{\"parameters\":{\"branch\":{\"defaultValue\":\"report\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"event_keywords\":{\"defaultValue\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"github_token\":{\"defaultValue\":\"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\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"log_keywords\":{\"defaultValue\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"isOptional\":true,\"parameterType\":\"STRING\"},\"repo_url\":{\"defaultValue\":\"https://github.com/Rosa1026/Kubeflow.git\",\"isOptional\":true,\"parameterType\":\"STRING\"}}}}","--task","","--runtime_config","{\"parameterValues\":{\"branch\":\"report\",\"event_keywords\":\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\",\"github_token\":\"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\",\"log_keywords\":\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\",\"repo_url\":\"https://github.com/Rosa1026/Kubeflow.git\"}}","--iteration_index","-1","--execution_id_path","/tmp/outputs/execution-id","--iteration_count_path","/tmp/outputs/iteration-count","--condition_path","/tmp/outputs/condition"],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-pipeline-advance-version-zdlvz/2025/05/13/pod-log-pipeline-advance-version-zdlvz-system-dag-driver-1383398674"}}}"defaultValue": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", "log_keywords": "CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready", I0513 08:58:36.380194 15 driver.go:185] Created execution: id:227 name:"run/d0e6f244-ec8b-46a8-ba5c-18eebf6904f9" type_id:13 type:"system.DAGExecution" last_known_state:RUNNING custom_properties:{key:"display_name" value:{string_value:""}} custom_properties:{key:"inputs" value:{struct_value:{fields:{key:"branch" value:{string_value:"report"}} fields:{key:"event_keywords" value:{string_value:"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory"}} fields:{key:"github_token" value:{string_value:"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"}} fields:{key:"log_keywords" value:{string_value:"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready"}} fields:{key:"repo_url" value:{string_value:"https://github.com/Rosa1026/Kubeflow.git"}}}}} custom_properties:{key:"task_name" value:{string_value:""}} create_time_since_epoch:1747126716346 last_update_time_since_epoch:1747126716346
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"159"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-to-github","--run_id","a797e690-1ae3-447c-9342-aaa288deef9a","--dag_execution_id","159","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-to-github-mxzg4/2025/05/13/pod-log-to-github-mxzg4-system-container-driver-451000317"}}}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"159"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-to-github","--run_id","a797e690-1ae3-447c-9342-aaa288deef9a","--dag_execution_id","159","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-to-github-mxzg4/2025/05/13/pod-log-to-github-mxzg4-system-container-driver-451000317"}}}
{"args":["--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset]):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 키워드 목록\n status_keywords = [\"CrashLoopBackOff\", \"Error\", \"Failed\", \"ImagePullBackOff\", \"Pending\"]\n describe_keywords = [\"CrashLoopBackOff\", \"OOMKilled\", \"Error\", \"Failed\", \"Back-off\"]\n log_keywords = [\"Traceback\", \"panic\", \"exception\", \"fail\", \"error\"]\n event_keywords = [\"Failed\", \"BackOff\", \"Unhealthy\", \"Killing\", \"Evicted\"]\n\n # kubectl 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n # Get pod status\n pods_output = subprocess.getoutput(\n \"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\"\n )\n pod_lines = pods_output.strip().split(\"\\n\")\n\n def match_keywords(lines, keywords):\n return \"\\n\".join([line for line in lines if any(k in line for k in keywords)])\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n if not any(k in status for k in status_keywords):\n continue # 중요 상태가 아닐 경우 무시\n\n describe_output = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n log_output = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\n log_lines = log_output.strip().splitlines()\n last_logs = \"\\n\".join(log_lines[-10:]) # 마지막 10줄만\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\n\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\u003c/h2\u003e\")\n if matched_describe:\n f.write(\"\u003ch3\u003eDescribe (filtered)\u003c/h3\u003e\u003cpre\u003e\" + matched_describe + \"\u003c/pre\u003e\")\n if matched_logs:\n f.write(\"\u003ch3\u003eLogs (last 10 lines, filtered)\u003c/h3\u003e\u003cpre\u003e\" + matched_logs + \"\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events_output = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"],"image":"python:3.9"}
ARGO_TEMPLATE: {"name":"system-container-driver","inputs":{"parameters":[{"name":"component","value":"{\"executorLabel\":\"exec-collect-filtered-logs\",\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}"},{"name":"task","value":"{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}"},{"name":"container","value":"{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}"},{"name":"parent-dag-id","value":"159"},{"name":"iteration-index","default":"-1","value":"-1"},{"name":"kubernetes-config","default":"","value":""}]},"outputs":{"parameters":[{"name":"pod-spec-patch","valueFrom":{"path":"/tmp/outputs/pod-spec-patch","default":""}},{"name":"cached-decision","default":"false","valueFrom":{"path":"/tmp/outputs/cached-decision","default":"false"}},{"name":"condition","valueFrom":{"path":"/tmp/outputs/condition","default":"true"}}]},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec","command":["driver"],"args":["--type","CONTAINER","--pipeline_name","pod-log-to-github","--run_id","a797e690-1ae3-447c-9342-aaa288deef9a","--dag_execution_id","159","--component","{\"executorLabel\":\"exec-collect-filtered-logs\",\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\",\"schemaVersion\":\"0.0.1\"}}}}}","--task","{\"cachingOptions\":{},\"componentRef\":{\"name\":\"comp-collect-filtered-logs\"},\"taskInfo\":{\"name\":\"collect-filtered-logs\"}}","--container","{\"args\":[\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"command\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\"],\"image\":\"python:3.9\"}","--iteration_index","-1","--cached_decision_path","/tmp/outputs/cached-decision","--pod_spec_patch_path","/tmp/outputs/pod-spec-patch","--condition_path","/tmp/outputs/condition","--kubernetes_config",""],"resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}},"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-to-github-mxzg4/2025/05/13/pod-log-to-github-mxzg4-system-container-driver-451000317"}}} "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset]):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 키워드 목록\n status_keywords = [\"CrashLoopBackOff\", \"Error\", \"Failed\", \"ImagePullBackOff\", \"Pending\"]\n describe_keywords = [\"CrashLoopBackOff\", \"OOMKilled\", \"Error\", \"Failed\", \"Back-off\"]\n log_keywords = [\"Traceback\", \"panic\", \"exception\", \"fail\", \"error\"]\n event_keywords = [\"Failed\", \"BackOff\", \"Unhealthy\", \"Killing\", \"Evicted\"]\n\n # kubectl 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n # Get pod status\n pods_output = subprocess.getoutput(\n \"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\"\n )\n pod_lines = pods_output.strip().split(\"\\n\")\n\n def match_keywords(lines, keywords):\n return \"\\n\".join([line for line in lines if any(k in line for k in keywords)])\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n if not any(k in status for k in status_keywords):\n continue # 중요 상태가 아닐 경우 무시\n\n describe_output = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n log_output = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\n log_lines = log_output.strip().splitlines()\n last_logs = \"\\n\".join(log_lines[-10:]) # 마지막 10줄만\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\n\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\u003c/h2\u003e\")\n if matched_describe:\n f.write(\"\u003ch3\u003eDescribe (filtered)\u003c/h3\u003e\u003cpre\u003e\" + matched_describe + \"\u003c/pre\u003e\")\n if matched_logs:\n f.write(\"\u003ch3\u003eLogs (last 10 lines, filtered)\u003c/h3\u003e\u003cpre\u003e\" + matched_logs + \"\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events_output = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n"
{"containers":[{"name":"main","image":"python:3.9","command":["/var/run/argo/argoexec","emissary","--","/kfp-launcher/launch","--pipeline_name","pod-log-to-github","--run_id","a797e690-1ae3-447c-9342-aaa288deef9a","--execution_id","160","--executor_input","{\"inputs\":{}, \"outputs\":{\"artifacts\":{\"output_report\":{\"artifacts\":[{\"type\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}, \"uri\":\"minio://mlpipeline/v2/artifacts/pod-log-to-github/a797e690-1ae3-447c-9342-aaa288deef9a/collect-filtered-logs/output_report\"}]}}, \"outputFile\":\"/tmp/kfp_outputs/output_metadata.json\"}}","--component_spec","{\"outputDefinitions\":{\"artifacts\":{\"output_report\":{\"artifactType\":{\"schemaTitle\":\"system.Dataset\", \"schemaVersion\":\"0.0.1\"}}}}, \"executorLabel\":\"exec-collect-filtered-logs\"}","--pod_name","$(KFP_POD_NAME)","--pod_uid","$(KFP_POD_UID)","--mlmd_server_address","$(METADATA_GRPC_SERVICE_HOST)","--mlmd_server_port","$(METADATA_GRPC_SERVICE_PORT)","--"],"args":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef collect_filtered_logs(output_report: Output[Dataset]):\n import subprocess\n import os\n\n tmp_report_path = \"/tmp/report.html\"\n seen_namespaces = set()\n namespace_events = {}\n\n # 키워드 목록\n status_keywords = [\"CrashLoopBackOff\", \"Error\", \"Failed\", \"ImagePullBackOff\", \"Pending\"]\n describe_keywords = [\"CrashLoopBackOff\", \"OOMKilled\", \"Error\", \"Failed\", \"Back-off\"]\n log_keywords = [\"Traceback\", \"panic\", \"exception\", \"fail\", \"error\"]\n event_keywords = [\"Failed\", \"BackOff\", \"Unhealthy\", \"Killing\", \"Evicted\"]\n\n # kubectl 설치\n subprocess.run([\"apt-get\", \"update\", \"-y\"], check=True)\n subprocess.run([\n \"curl\", \"-LO\", \"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\"\n ], check=True)\n subprocess.run([\"chmod\", \"+x\", \"kubectl\"], check=True)\n subprocess.run([\"mv\", \"kubectl\", \"/usr/local/bin/\"], check=True)\n\n # Get pod status\n pods_output = subprocess.getoutput(\n \"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\"\n )\n pod_lines = pods_output.strip().split(\"\\n\")\n\n def match_keywords(lines, keywords):\n return \"\\n\".join([line for line in lines if any(k in line for k in keywords)])\n\n with open(tmp_report_path, \"w\") as f:\n f.write(\"\"\"\n \u003chtml\u003e\n \u003chead\u003e\n \u003cmeta charset=\"UTF-8\"\u003e\n \u003cstyle\u003e\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\n h1 { color: #2c3e50; }\n h2 { color: #34495e; }\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\n section { margin-bottom: 30px; }\n \u003c/style\u003e\n \u003c/head\u003e\n \u003cbody\u003e\n \u003ch1\u003eKubernetes 장애 점검 리포트\u003c/h1\u003e\n \"\"\")\n\n for line in pod_lines:\n if not line.strip():\n continue\n try:\n namespace, pod_name, status = line.strip().split()\n except ValueError:\n continue\n\n if not any(k in status for k in status_keywords):\n continue # 중요 상태가 아닐 경우 무시\n\n describe_output = subprocess.getoutput(f\"kubectl describe pod {pod_name} -n {namespace}\")\n log_output = subprocess.getoutput(f\"kubectl logs {pod_name} -n {namespace}\")\n\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\n log_lines = log_output.strip().splitlines()\n last_logs = \"\\n\".join(log_lines[-10:]) # 마지막 10줄만\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\n\n f.write(f\"\u003csection\u003e\u003ch2\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\u003c/h2\u003e\")\n if matched_describe:\n f.write(\"\u003ch3\u003eDescribe (filtered)\u003c/h3\u003e\u003cpre\u003e\" + matched_describe + \"\u003c/pre\u003e\")\n if matched_logs:\n f.write(\"\u003ch3\u003eLogs (last 10 lines, filtered)\u003c/h3\u003e\u003cpre\u003e\" + matched_logs + \"\u003c/pre\u003e\")\n f.write(\"\u003c/section\u003e\")\n\n if namespace not in seen_namespaces:\n events_output = subprocess.getoutput(f\"kubectl get events -n {namespace}\")\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\n if matched_events:\n namespace_events[namespace] = matched_events\n seen_namespaces.add(namespace)\n\n for ns, events in namespace_events.items():\n f.write(f\"\u003csection\u003e\u003ch2\u003eEvents in Namespace: {ns}\u003c/h2\u003e\u003cpre\u003e{events}\u003c/pre\u003e\u003c/section\u003e\")\n\n f.write(\"\u003c/body\u003e\u003c/html\u003e\")\n\n subprocess.run([\"cp\", tmp_report_path, output_report.path], check=True)\n\n","--executor_input","{{$}}","--function_to_execute","collect_filtered_logs"],"resources":{}}]} ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-to-github\",\"--run_id\",\"a797e690-1ae3-447c-9342-aaa288deef9a\",\"--execution_id\",\"160\",\"--executor_input\",\"{\\\"inputs\\\":{}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-to-github/a797e690-1ae3-447c-9342-aaa288deef9a/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-to-github-mxzg4/2025/05/13/pod-log-to-github-mxzg4-system-container-impl-169698603"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-to-github\",\"--run_id\",\"a797e690-1ae3-447c-9342-aaa288deef9a\",\"--execution_id\",\"160\",\"--executor_input\",\"{\\\"inputs\\\":{}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-to-github/a797e690-1ae3-447c-9342-aaa288deef9a/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-to-github\",\"--run_id\",\"a797e690-1ae3-447c-9342-aaa288deef9a\",\"--execution_id\",\"160\",\"--executor_input\",\"{\\\"inputs\\\":{}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-to-github/a797e690-1ae3-447c-9342-aaa288deef9a/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-to-github-mxzg4/2025/05/13/pod-log-to-github-mxzg4-system-container-impl-169698603"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-to-github\",\"--run_id\",\"a797e690-1ae3-447c-9342-aaa288deef9a\",\"--execution_id\",\"160\",\"--executor_input\",\"{\\\"inputs\\\":{}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-to-github/a797e690-1ae3-447c-9342-aaa288deef9a/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-to-github\",\"--run_id\",\"a797e690-1ae3-447c-9342-aaa288deef9a\",\"--execution_id\",\"160\",\"--executor_input\",\"{\\\"inputs\\\":{}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-to-github/a797e690-1ae3-447c-9342-aaa288deef9a/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-to-github-mxzg4/2025/05/13/pod-log-to-github-mxzg4-system-container-impl-169698603"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-to-github\",\"--run_id\",\"a797e690-1ae3-447c-9342-aaa288deef9a\",\"--execution_id\",\"160\",\"--executor_input\",\"{\\\"inputs\\\":{}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-to-github/a797e690-1ae3-447c-9342-aaa288deef9a/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}
status_keywords = ["CrashLoopBackOff", "Error", "Failed", "ImagePullBackOff", "Pending"]
describe_keywords = ["CrashLoopBackOff", "OOMKilled", "Error", "Failed", "Back-off"]
ARGO_TEMPLATE: {"name":"system-container-impl","inputs":{"parameters":[{"name":"pod-spec-patch","value":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-to-github\",\"--run_id\",\"a797e690-1ae3-447c-9342-aaa288deef9a\",\"--execution_id\",\"160\",\"--executor_input\",\"{\\\"inputs\\\":{}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-to-github/a797e690-1ae3-447c-9342-aaa288deef9a/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}]},"outputs":{},"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"container":{"name":"","image":"gcr.io/ml-pipeline/should-be-overridden-during-runtime","command":["should-be-overridden-during-runtime"],"envFrom":[{"configMapRef":{"name":"metadata-grpc-configmap","optional":true}}],"env":[{"name":"KFP_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"KFP_POD_UID","valueFrom":{"fieldRef":{"fieldPath":"metadata.uid"}}}],"resources":{},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]},"volumes":[{"name":"kfp-launcher","emptyDir":{}}],"initContainers":[{"name":"kfp-launcher","image":"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f","command":["launcher-v2","--copy","/kfp-launcher/launch"],"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kfp-launcher","mountPath":"/kfp-launcher"}]}],"archiveLocation":{"archiveLogs":true,"s3":{"endpoint":"minio-service.kubeflow:9000","bucket":"mlpipeline","insecure":true,"accessKeySecret":{"name":"mlpipeline-minio-artifact","key":"accesskey"},"secretKeySecret":{"name":"mlpipeline-minio-artifact","key":"secretkey"},"key":"artifacts/pod-log-to-github-mxzg4/2025/05/13/pod-log-to-github-mxzg4-system-container-impl-169698603"}},"podSpecPatch":"{\"containers\":[{\"name\":\"main\",\"image\":\"python:3.9\",\"command\":[\"/var/run/argo/argoexec\",\"emissary\",\"--\",\"/kfp-launcher/launch\",\"--pipeline_name\",\"pod-log-to-github\",\"--run_id\",\"a797e690-1ae3-447c-9342-aaa288deef9a\",\"--execution_id\",\"160\",\"--executor_input\",\"{\\\"inputs\\\":{}, \\\"outputs\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifacts\\\":[{\\\"type\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}, \\\"uri\\\":\\\"minio://mlpipeline/v2/artifacts/pod-log-to-github/a797e690-1ae3-447c-9342-aaa288deef9a/collect-filtered-logs/output_report\\\"}]}}, \\\"outputFile\\\":\\\"/tmp/kfp_outputs/output_metadata.json\\\"}}\",\"--component_spec\",\"{\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\", \\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}, \\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\"}\",\"--pod_name\",\"$(KFP_POD_NAME)\",\"--pod_uid\",\"$(KFP_POD_UID)\",\"--mlmd_server_address\",\"$(METADATA_GRPC_SERVICE_HOST)\",\"--mlmd_server_port\",\"$(METADATA_GRPC_SERVICE_PORT)\",\"--\"],\"args\":[\"sh\",\"-c\",\"\\nif ! [ -x \\\"$(command -v pip)\\\" ]; then\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\nfi\\n\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\u003e=3.7.4,\\u003c5; python_version\\u003c\\\"3.9\\\"' \\u0026\\u0026 \\\"$0\\\" \\\"$@\\\"\\n\",\"sh\",\"-ec\",\"program_path=$(mktemp -d)\\n\\nprintf \\\"%s\\\" \\\"$0\\\" \\u003e \\\"$program_path/ephemeral_component.py\\\"\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\"$program_path/ephemeral_component.py\\\" \\\"$@\\\"\\n\",\"\\nimport kfp\\nfrom kfp import dsl\\nfrom kfp.dsl import *\\nfrom typing import *\\n\\ndef collect_filtered_logs(output_report: Output[Dataset]):\\n import subprocess\\n import os\\n\\n tmp_report_path = \\\"/tmp/report.html\\\"\\n seen_namespaces = set()\\n namespace_events = {}\\n\\n # 키워드 목록\\n status_keywords = [\\\"CrashLoopBackOff\\\", \\\"Error\\\", \\\"Failed\\\", \\\"ImagePullBackOff\\\", \\\"Pending\\\"]\\n describe_keywords = [\\\"CrashLoopBackOff\\\", \\\"OOMKilled\\\", \\\"Error\\\", \\\"Failed\\\", \\\"Back-off\\\"]\\n log_keywords = [\\\"Traceback\\\", \\\"panic\\\", \\\"exception\\\", \\\"fail\\\", \\\"error\\\"]\\n event_keywords = [\\\"Failed\\\", \\\"BackOff\\\", \\\"Unhealthy\\\", \\\"Killing\\\", \\\"Evicted\\\"]\\n\\n # kubectl 설치\\n subprocess.run([\\\"apt-get\\\", \\\"update\\\", \\\"-y\\\"], check=True)\\n subprocess.run([\\n \\\"curl\\\", \\\"-LO\\\", \\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\"\\n ], check=True)\\n subprocess.run([\\\"chmod\\\", \\\"+x\\\", \\\"kubectl\\\"], check=True)\\n subprocess.run([\\\"mv\\\", \\\"kubectl\\\", \\\"/usr/local/bin/\\\"], check=True)\\n\\n # Get pod status\\n pods_output = subprocess.getoutput(\\n \\\"kubectl get pods --all-namespaces -o=custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers\\\"\\n )\\n pod_lines = pods_output.strip().split(\\\"\\\\n\\\")\\n\\n def match_keywords(lines, keywords):\\n return \\\"\\\\n\\\".join([line for line in lines if any(k in line for k in keywords)])\\n\\n with open(tmp_report_path, \\\"w\\\") as f:\\n f.write(\\\"\\\"\\\"\\n \\u003chtml\\u003e\\n \\u003chead\\u003e\\n \\u003cmeta charset=\\\"UTF-8\\\"\\u003e\\n \\u003cstyle\\u003e\\n body { font-family: Arial, sans-serif; background-color: #f4f4f4; padding: 20px; }\\n h1 { color: #2c3e50; }\\n h2 { color: #34495e; }\\n pre { background-color: #ecf0f1; padding: 10px; border-left: 4px solid #3498db; overflow-x: auto; }\\n section { margin-bottom: 30px; }\\n \\u003c/style\\u003e\\n \\u003c/head\\u003e\\n \\u003cbody\\u003e\\n \\u003ch1\\u003eKubernetes 장애 점검 리포트\\u003c/h1\\u003e\\n \\\"\\\"\\\")\\n\\n for line in pod_lines:\\n if not line.strip():\\n continue\\n try:\\n namespace, pod_name, status = line.strip().split()\\n except ValueError:\\n continue\\n\\n if not any(k in status for k in status_keywords):\\n continue # 중요 상태가 아닐 경우 무시\\n\\n describe_output = subprocess.getoutput(f\\\"kubectl describe pod {pod_name} -n {namespace}\\\")\\n log_output = subprocess.getoutput(f\\\"kubectl logs {pod_name} -n {namespace}\\\")\\n\\n matched_describe = match_keywords(describe_output.splitlines(), describe_keywords)\\n log_lines = log_output.strip().splitlines()\\n last_logs = \\\"\\\\n\\\".join(log_lines[-10:]) # 마지막 10줄만\\n matched_logs = match_keywords(last_logs.splitlines(), log_keywords)\\n\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003ePod: {pod_name} (Namespace: {namespace}, Status: {status})\\u003c/h2\\u003e\\\")\\n if matched_describe:\\n f.write(\\\"\\u003ch3\\u003eDescribe (filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_describe + \\\"\\u003c/pre\\u003e\\\")\\n if matched_logs:\\n f.write(\\\"\\u003ch3\\u003eLogs (last 10 lines, filtered)\\u003c/h3\\u003e\\u003cpre\\u003e\\\" + matched_logs + \\\"\\u003c/pre\\u003e\\\")\\n f.write(\\\"\\u003c/section\\u003e\\\")\\n\\n if namespace not in seen_namespaces:\\n events_output = subprocess.getoutput(f\\\"kubectl get events -n {namespace}\\\")\\n matched_events = match_keywords(events_output.splitlines(), event_keywords)\\n if matched_events:\\n namespace_events[namespace] = matched_events\\n seen_namespaces.add(namespace)\\n\\n for ns, events in namespace_events.items():\\n f.write(f\\\"\\u003csection\\u003e\\u003ch2\\u003eEvents in Namespace: {ns}\\u003c/h2\\u003e\\u003cpre\\u003e{events}\\u003c/pre\\u003e\\u003c/section\\u003e\\\")\\n\\n f.write(\\\"\\u003c/body\\u003e\\u003c/html\\u003e\\\")\\n\\n subprocess.run([\\\"cp\\\", tmp_report_path, output_report.path], check=True)\\n\\n\",\"--executor_input\",\"{{$}}\",\"--function_to_execute\",\"collect_filtered_logs\"],\"resources\":{}}]}"}time="2025-05-14T00:20:42Z" level=error msg="Transient failure while syncing resource (kubeflow-user-example-com/pod-log-pipeline-advance-version-r84g4): CustomError (code: 0): Syncing Workflow (pod-log-pipeline-advance-version-r84g4): transient failure: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-r84g4\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"ca7c0d8d-aa00-4f03-8ed2-a8ddc7cc8385\",\"resourceVersion\":\"9339045\",\"generation\":5,\"creationTimestamp\":\"2025-05-14T00:19:57Z\",\"labels\":{\"pipeline/runid\":\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:19:57Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:20:27Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"nodes\":{\"pod-log-pipeline-advance-version-r84g4\":{\"id\":\"pod-log-pipeline-advance-version-r84g4\",\"name\":\"pod-log-pipeline-advance-version-r84g4\",\"displayName\":\"pod-log-pipeline-advance-version-r84g4\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"children\":[\"pod-log-pipeline-advance-version-r84g4-2332657773\"]},\"pod-log-pipeline-advance-version-r84g4-1580615280\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root\",\"displayName\":\"root\",\"type\":\"DAG\",\"templateName\":\"root\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4\",\"startedAt\":\"2025-05-14T00:20:07Z\",\"finishedAt\":null,\"progress\":\"1/2\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"229\"}]},\"children\":[\"pod-log-pipeline-advance-version-r84g4-3520029909\"]},\"pod-log-pipeline-advance-version-r84g4-1731727560\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-1731727560\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs\",\"displayName\":\"collect-filtered-logs\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:20:17Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-r84g4-2405826195\"]},\"pod-log-pipeline-advance-version-r84g4-2332657773\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-2332657773\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":\"2025-05-14T00:20:00Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":5,\"memory\":4},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"value\":\"229\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"value\":\"0\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-dag-driver-2332657773/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-1580615280\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-2405826195\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-2405826195\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1731727560\",\"startedAt\":\"2025-05-14T00:20:17Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"}]},\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-3520029909\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-3520029909\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs-driver\",\"displayName\":\"collect-filtered-logs-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:20:07Z\",\"finishedAt\":\"2025-05-14T00:20:11Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":5,\"memory\":4},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"229\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-driver-3520029909/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-1731727560\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"}},\"conditions\":[{\"type\":\"PodRunning\",\"status\":\"True\"}],\"resourcesDuration\":{\"cpu\":10,\"memory\":8},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-r84g4\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"ca7c0d8d-aa00-4f03-8ed2-a8ddc7cc8385\",\"resourceVersion\":\"9339045\",\"generation\":5,\"creationTimestamp\":\"2025-05-14T00:19:57Z\",\"labels\":{\"pipeline/runid\":\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:19:57Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:20:27Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"nodes\":{\"pod-log-pipeline-advance-version-r84g4\":{\"id\":\"pod-log-pipeline-advance-version-r84g4\",\"name\":\"pod-log-pipeline-advance-version-r84g4\",\"displayName\":\"pod-log-pipeline-advance-version-r84g4\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"children\":[\"pod-log-pipeline-advance-version-r84g4-2332657773\"]},\"pod-log-pipeline-advance-version-r84g4-1580615280\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root\",\"displayName\":\"root\",\"type\":\"DAG\",\"templateName\":\"root\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4\",\"startedAt\":\"2025-05-14T00:20:07Z\",\"finishedAt\":null,\"progress\":\"1/2\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"229\"}]},\"children\":[\"pod-log-pipeline-advance-version-r84g4-3520029909\"]},\"pod-log-pipeline-advance-version-r84g4-1731727560\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-1731727560\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs\",\"displayName\":\"collect-filtered-logs\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:20:17Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-r84g4-2405826195\"]},\"pod-log-pipeline-advance-version-r84g4-2332657773\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-2332657773\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":\"2025-05-14T00:20:00Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":5,\"memory\":4},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"value\":\"229\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"value\":\"0\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-dag-driver-2332657773/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-1580615280\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-2405826195\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-2405826195\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1731727560\",\"startedAt\":\"2025-05-14T00:20:17Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"}]},\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-3520029909\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-3520029909\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs-driver\",\"displayName\":\"collect-filtered-logs-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:20:07Z\",\"finishedAt\":\"2025-05-14T00:20:11Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":5,\"memory\":4},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"229\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-driver-3520029909/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-1731727560\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"}},\"conditions\":[{\"type\":\"PodRunning\",\"status\":\"True\"}],\"resourcesDuration\":{\"cpu\":10,\"memory\":8},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired"
time="2025-05-14T00:23:42Z" level=error msg="Transient failure while syncing resource (kubeflow-user-example-com/pod-log-pipeline-advance-version-5nw44): CustomError (code: 0): Syncing Workflow (pod-log-pipeline-advance-version-5nw44): transient failure: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"f589cec9-5545-41fe-87ab-a38e6790cb33\",\"resourceVersion\":\"9340175\",\"generation\":2,\"creationTimestamp\":\"2025-05-14T00:23:41Z\",\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"nodes\":{\"pod-log-pipeline-advance-version-5nw44\":{\"id\":\"pod-log-pipeline-advance-version-5nw44\",\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"displayName\":\"pod-log-pipeline-advance-version-5nw44\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"children\":[\"pod-log-pipeline-advance-version-5nw44-92227836\"]},\"pod-log-pipeline-advance-version-5nw44-92227836\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-92227836\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]}}},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"f589cec9-5545-41fe-87ab-a38e6790cb33\",\"resourceVersion\":\"9340175\",\"generation\":2,\"creationTimestamp\":\"2025-05-14T00:23:41Z\",\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"nodes\":{\"pod-log-pipeline-advance-version-5nw44\":{\"id\":\"pod-log-pipeline-advance-version-5nw44\",\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"displayName\":\"pod-log-pipeline-advance-version-5nw44\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"children\":[\"pod-log-pipeline-advance-version-5nw44-92227836\"]},\"pod-log-pipeline-advance-version-5nw44-92227836\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-92227836\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]}}},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired"
time="2025-05-14T00:23:50Z" level=error msg="Transient failure while syncing resource (kubeflow-user-example-com/pod-log-pipeline-advance-version-5nw44): CustomError (code: 0): Syncing Workflow (pod-log-pipeline-advance-version-5nw44): transient failure: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"f589cec9-5545-41fe-87ab-a38e6790cb33\",\"resourceVersion\":\"9340175\",\"generation\":2,\"creationTimestamp\":\"2025-05-14T00:23:41Z\",\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"nodes\":{\"pod-log-pipeline-advance-version-5nw44\":{\"id\":\"pod-log-pipeline-advance-version-5nw44\",\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"displayName\":\"pod-log-pipeline-advance-version-5nw44\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"children\":[\"pod-log-pipeline-advance-version-5nw44-92227836\"]},\"pod-log-pipeline-advance-version-5nw44-92227836\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-92227836\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]}}},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"f589cec9-5545-41fe-87ab-a38e6790cb33\",\"resourceVersion\":\"9340175\",\"generation\":2,\"creationTimestamp\":\"2025-05-14T00:23:41Z\",\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"nodes\":{\"pod-log-pipeline-advance-version-5nw44\":{\"id\":\"pod-log-pipeline-advance-version-5nw44\",\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"displayName\":\"pod-log-pipeline-advance-version-5nw44\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"children\":[\"pod-log-pipeline-advance-version-5nw44-92227836\"]},\"pod-log-pipeline-advance-version-5nw44-92227836\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-92227836\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]}}},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired"
time="2025-05-14T00:24:06Z" level=error msg="Transient failure while syncing resource (kubeflow-user-example-com/pod-log-pipeline-advance-version-5nw44): CustomError (code: 0): Syncing Workflow (pod-log-pipeline-advance-version-5nw44): transient failure: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"f589cec9-5545-41fe-87ab-a38e6790cb33\",\"resourceVersion\":\"9340325\",\"generation\":4,\"creationTimestamp\":\"2025-05-14T00:23:41Z\",\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:24:01Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"nodes\":{\"pod-log-pipeline-advance-version-5nw44\":{\"id\":\"pod-log-pipeline-advance-version-5nw44\",\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"displayName\":\"pod-log-pipeline-advance-version-5nw44\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"children\":[\"pod-log-pipeline-advance-version-5nw44-92227836\"]},\"pod-log-pipeline-advance-version-5nw44-2407325076\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-2407325076\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs-driver\",\"displayName\":\"collect-filtered-logs-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:23:51Z\",\"finishedAt\":\"2025-05-14T00:23:55Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"232\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-driver-2407325076/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-298533763\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-5nw44-2856536475\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root\",\"displayName\":\"root\",\"type\":\"DAG\",\"templateName\":\"root\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:51Z\",\"finishedAt\":null,\"progress\":\"1/2\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"232\"}]},\"children\":[\"pod-log-pipeline-advance-version-5nw44-2407325076\"]},\"pod-log-pipeline-advance-version-5nw44-298533763\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-298533763\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs\",\"displayName\":\"collect-filtered-logs\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:24:01Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-5nw44-4123328826\"]},\"pod-log-pipeline-advance-version-5nw44-4123328826\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-4123328826\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-298533763\",\"startedAt\":\"2025-05-14T00:24:01Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"}]}},\"pod-log-pipeline-advance-version-5nw44-92227836\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-92227836\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":\"2025-05-14T00:23:45Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"value\":\"232\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"value\":\"0\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-dag-driver-92227836/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-2856536475\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"}},\"conditions\":[{\"type\":\"PodRunning\",\"status\":\"False\"}],\"resourcesDuration\":{\"cpu\":8,\"memory\":6},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"f589cec9-5545-41fe-87ab-a38e6790cb33\",\"resourceVersion\":\"9340325\",\"generation\":4,\"creationTimestamp\":\"2025-05-14T00:23:41Z\",\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:24:01Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"nodes\":{\"pod-log-pipeline-advance-version-5nw44\":{\"id\":\"pod-log-pipeline-advance-version-5nw44\",\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"displayName\":\"pod-log-pipeline-advance-version-5nw44\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"children\":[\"pod-log-pipeline-advance-version-5nw44-92227836\"]},\"pod-log-pipeline-advance-version-5nw44-2407325076\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-2407325076\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs-driver\",\"displayName\":\"collect-filtered-logs-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:23:51Z\",\"finishedAt\":\"2025-05-14T00:23:55Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"232\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-driver-2407325076/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-298533763\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-5nw44-2856536475\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root\",\"displayName\":\"root\",\"type\":\"DAG\",\"templateName\":\"root\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:51Z\",\"finishedAt\":null,\"progress\":\"1/2\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"232\"}]},\"children\":[\"pod-log-pipeline-advance-version-5nw44-2407325076\"]},\"pod-log-pipeline-advance-version-5nw44-298533763\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-298533763\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs\",\"displayName\":\"collect-filtered-logs\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:24:01Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-5nw44-4123328826\"]},\"pod-log-pipeline-advance-version-5nw44-4123328826\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-4123328826\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-298533763\",\"startedAt\":\"2025-05-14T00:24:01Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"}]}},\"pod-log-pipeline-advance-version-5nw44-92227836\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-92227836\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":\"2025-05-14T00:23:45Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"value\":\"232\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"value\":\"0\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-dag-driver-92227836/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-2856536475\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"}},\"conditions\":[{\"type\":\"PodRunning\",\"status\":\"False\"}],\"resourcesDuration\":{\"cpu\":8,\"memory\":6},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired"
time="2025-05-14T00:26:12Z" level=error msg="Transient failure while syncing resource (kubeflow-user-example-com/pod-log-pipeline-advance-version-xr85j): CustomError (code: 0): Syncing Workflow (pod-log-pipeline-advance-version-xr85j): transient failure: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"8af6b44a-3a5d-40d2-bcfa-91ed64bd59f0\",\"resourceVersion\":\"9341166\",\"generation\":2,\"creationTimestamp\":\"2025-05-14T00:26:11Z\",\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:11Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:11Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"nodes\":{\"pod-log-pipeline-advance-version-xr85j\":{\"id\":\"pod-log-pipeline-advance-version-xr85j\",\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"displayName\":\"pod-log-pipeline-advance-version-xr85j\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"children\":[\"pod-log-pipeline-advance-version-xr85j-4218789953\"]},\"pod-log-pipeline-advance-version-xr85j-4218789953\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-4218789953\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]}}},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"8af6b44a-3a5d-40d2-bcfa-91ed64bd59f0\",\"resourceVersion\":\"9341166\",\"generation\":2,\"creationTimestamp\":\"2025-05-14T00:26:11Z\",\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:11Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:11Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"nodes\":{\"pod-log-pipeline-advance-version-xr85j\":{\"id\":\"pod-log-pipeline-advance-version-xr85j\",\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"displayName\":\"pod-log-pipeline-advance-version-xr85j\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"children\":[\"pod-log-pipeline-advance-version-xr85j-4218789953\"]},\"pod-log-pipeline-advance-version-xr85j-4218789953\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-4218789953\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]}}},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired"
time="2025-05-14T00:26:14Z" level=error msg="Transient failure while syncing resource (kubeflow-user-example-com/pod-log-pipeline-advance-version-5nw44): CustomError (code: 0): Syncing Workflow (pod-log-pipeline-advance-version-5nw44): transient failure: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"f589cec9-5545-41fe-87ab-a38e6790cb33\",\"resourceVersion\":\"9341090\",\"generation\":9,\"creationTimestamp\":\"2025-05-14T00:23:41Z\",\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"workflows.argoproj.io/completed\":\"true\",\"workflows.argoproj.io/phase\":\"Failed\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:25:59Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/completed\":{},\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Failed\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":\"2025-05-14T00:25:59Z\",\"progress\":\"4/5\",\"nodes\":{\"pod-log-pipeline-advance-version-5nw44\":{\"id\":\"pod-log-pipeline-advance-version-5nw44\",\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"displayName\":\"pod-log-pipeline-advance-version-5nw44\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Failed\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":\"2025-05-14T00:25:59Z\",\"progress\":\"4/5\",\"resourcesDuration\":{\"cpu\":175,\"memory\":172},\"children\":[\"pod-log-pipeline-advance-version-5nw44-92227836\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-5nw44-911930307\"]},\"pod-log-pipeline-advance-version-5nw44-2312877464\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-2312877464\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.upload-to-github\",\"displayName\":\"upload-to-github\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:25:33Z\",\"finishedAt\":\"2025-05-14T00:25:59Z\",\"progress\":\"0/1\",\"resourcesDuration\":{\"cpu\":26,\"memory\":26},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"234\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"45\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-5nw44-911930307\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-5nw44-911930307\"]},\"pod-log-pipeline-advance-version-5nw44-2407325076\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-2407325076\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs-driver\",\"displayName\":\"collect-filtered-logs-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:23:51Z\",\"finishedAt\":\"2025-05-14T00:23:55Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"232\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-driver-2407325076/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-298533763\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-5nw44-2856536475\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root\",\"displayName\":\"root\",\"type\":\"DAG\",\"templateName\":\"root\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:51Z\",\"finishedAt\":\"2025-05-14T00:25:59Z\",\"progress\":\"3/4\",\"resourcesDuration\":{\"cpu\":171,\"memory\":169},\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"232\"}]},\"children\":[\"pod-log-pipeline-advance-version-5nw44-2407325076\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-5nw44-911930307\"]},\"pod-log-pipeline-advance-version-5nw44-298533763\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-298533763\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs\",\"displayName\":\"collect-filtered-logs\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:24:01Z\",\"finishedAt\":\"2025-05-14T00:25:23Z\",\"progress\":\"2/3\",\"resourcesDuration\":{\"cpu\":167,\"memory\":166},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-5nw44-4123328826\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-5nw44-4123328826\"]},\"pod-log-pipeline-advance-version-5nw44-4123328826\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-4123328826\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-298533763\",\"startedAt\":\"2025-05-14T00:24:01Z\",\"finishedAt\":\"2025-05-14T00:25:13Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":138,\"memory\":138},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"}]},\"outputs\":{\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-impl-4123328826/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-840634021\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-5nw44-840634021\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-840634021\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.upload-to-github-driver\",\"displayName\":\"upload-to-github-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:25:23Z\",\"finishedAt\":\"2025-05-14T00:25:27Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":3,\"memory\":2},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"232\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"234\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"45\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-driver-840634021/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-2312877464\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-5nw44-911930307\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-911930307\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.upload-to-github.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2312877464\",\"message\":\"Error (exit code 1)\",\"startedAt\":\"2025-05-14T00:25:33Z\",\"finishedAt\":\"2025-05-14T00:25:49Z\",\"progress\":\"0/1\",\"resourcesDuration\":{\"cpu\":26,\"memory\":26},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"234\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"45\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\"}]},\"outputs\":{\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-impl-911930307/main.log\"}}],\"exitCode\":\"1\"},\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-5nw44-92227836\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-92227836\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":\"2025-05-14T00:23:45Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"value\":\"232\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"value\":\"0\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-dag-driver-92227836/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-2856536475\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"}},\"conditions\":[{\"type\":\"PodRunning\",\"status\":\"False\"},{\"type\":\"Completed\",\"status\":\"True\"}],\"resourcesDuration\":{\"cpu\":175,\"memory\":172},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"f589cec9-5545-41fe-87ab-a38e6790cb33\",\"resourceVersion\":\"9341090\",\"generation\":9,\"creationTimestamp\":\"2025-05-14T00:23:41Z\",\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"workflows.argoproj.io/completed\":\"true\",\"workflows.argoproj.io/phase\":\"Failed\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:23:41Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:25:59Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/completed\":{},\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"2c4a0994-92f0-4682-9289-dda2e308be96\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Failed\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":\"2025-05-14T00:25:59Z\",\"progress\":\"4/5\",\"nodes\":{\"pod-log-pipeline-advance-version-5nw44\":{\"id\":\"pod-log-pipeline-advance-version-5nw44\",\"name\":\"pod-log-pipeline-advance-version-5nw44\",\"displayName\":\"pod-log-pipeline-advance-version-5nw44\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Failed\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":\"2025-05-14T00:25:59Z\",\"progress\":\"4/5\",\"resourcesDuration\":{\"cpu\":175,\"memory\":172},\"children\":[\"pod-log-pipeline-advance-version-5nw44-92227836\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-5nw44-911930307\"]},\"pod-log-pipeline-advance-version-5nw44-2312877464\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-2312877464\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.upload-to-github\",\"displayName\":\"upload-to-github\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:25:33Z\",\"finishedAt\":\"2025-05-14T00:25:59Z\",\"progress\":\"0/1\",\"resourcesDuration\":{\"cpu\":26,\"memory\":26},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"234\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"45\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-5nw44-911930307\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-5nw44-911930307\"]},\"pod-log-pipeline-advance-version-5nw44-2407325076\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-2407325076\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs-driver\",\"displayName\":\"collect-filtered-logs-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:23:51Z\",\"finishedAt\":\"2025-05-14T00:23:55Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"232\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-driver-2407325076/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-298533763\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-5nw44-2856536475\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root\",\"displayName\":\"root\",\"type\":\"DAG\",\"templateName\":\"root\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:51Z\",\"finishedAt\":\"2025-05-14T00:25:59Z\",\"progress\":\"3/4\",\"resourcesDuration\":{\"cpu\":171,\"memory\":169},\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"232\"}]},\"children\":[\"pod-log-pipeline-advance-version-5nw44-2407325076\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-5nw44-911930307\"]},\"pod-log-pipeline-advance-version-5nw44-298533763\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-298533763\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs\",\"displayName\":\"collect-filtered-logs\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:24:01Z\",\"finishedAt\":\"2025-05-14T00:25:23Z\",\"progress\":\"2/3\",\"resourcesDuration\":{\"cpu\":167,\"memory\":166},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-5nw44-4123328826\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-5nw44-4123328826\"]},\"pod-log-pipeline-advance-version-5nw44-4123328826\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-4123328826\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.collect-filtered-logs.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-298533763\",\"startedAt\":\"2025-05-14T00:24:01Z\",\"finishedAt\":\"2025-05-14T00:25:13Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":138,\"memory\":138},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"233\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"}]},\"outputs\":{\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-impl-4123328826/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-840634021\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-5nw44-840634021\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-840634021\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.upload-to-github-driver\",\"displayName\":\"upload-to-github-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2856536475\",\"startedAt\":\"2025-05-14T00:25:23Z\",\"finishedAt\":\"2025-05-14T00:25:27Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":3,\"memory\":2},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"232\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"234\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"45\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-driver-840634021/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-2312877464\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-5nw44-911930307\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-911930307\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root.upload-to-github.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44-2312877464\",\"message\":\"Error (exit code 1)\",\"startedAt\":\"2025-05-14T00:25:33Z\",\"finishedAt\":\"2025-05-14T00:25:49Z\",\"progress\":\"0/1\",\"resourcesDuration\":{\"cpu\":26,\"memory\":26},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"2c4a0994-92f0-4682-9289-dda2e308be96\\\",\\\"--execution_id\\\",\\\"234\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"45\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/2c4a0994-92f0-4682-9289-dda2e308be96/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\"}]},\"outputs\":{\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-container-impl-911930307/main.log\"}}],\"exitCode\":\"1\"},\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-5nw44-92227836\":{\"id\":\"pod-log-pipeline-advance-version-5nw44-92227836\",\"name\":\"pod-log-pipeline-advance-version-5nw44.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-5nw44\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-5nw44\",\"startedAt\":\"2025-05-14T00:23:41Z\",\"finishedAt\":\"2025-05-14T00:23:45Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"value\":\"232\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"value\":\"0\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-5nw44/2025/05/14/pod-log-pipeline-advance-version-5nw44-system-dag-driver-92227836/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-5nw44-2856536475\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"}},\"conditions\":[{\"type\":\"PodRunning\",\"status\":\"False\"},{\"type\":\"Completed\",\"status\":\"True\"}],\"resourcesDuration\":{\"cpu\":175,\"memory\":172},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired"
time="2025-05-14T00:26:20Z" level=error msg="Transient failure while syncing resource (kubeflow-user-example-com/pod-log-pipeline-advance-version-xr85j): CustomError (code: 0): Syncing Workflow (pod-log-pipeline-advance-version-xr85j): transient failure: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"8af6b44a-3a5d-40d2-bcfa-91ed64bd59f0\",\"resourceVersion\":\"9341166\",\"generation\":2,\"creationTimestamp\":\"2025-05-14T00:26:11Z\",\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:11Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:11Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"nodes\":{\"pod-log-pipeline-advance-version-xr85j\":{\"id\":\"pod-log-pipeline-advance-version-xr85j\",\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"displayName\":\"pod-log-pipeline-advance-version-xr85j\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"children\":[\"pod-log-pipeline-advance-version-xr85j-4218789953\"]},\"pod-log-pipeline-advance-version-xr85j-4218789953\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-4218789953\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]}}},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"8af6b44a-3a5d-40d2-bcfa-91ed64bd59f0\",\"resourceVersion\":\"9341166\",\"generation\":2,\"creationTimestamp\":\"2025-05-14T00:26:11Z\",\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:11Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:11Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"nodes\":{\"pod-log-pipeline-advance-version-xr85j\":{\"id\":\"pod-log-pipeline-advance-version-xr85j\",\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"displayName\":\"pod-log-pipeline-advance-version-xr85j\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"children\":[\"pod-log-pipeline-advance-version-xr85j-4218789953\"]},\"pod-log-pipeline-advance-version-xr85j-4218789953\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-4218789953\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]}}},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired"
time="2025-05-14T00:26:36Z" level=error msg="Transient failure while syncing resource (kubeflow-user-example-com/pod-log-pipeline-advance-version-xr85j): CustomError (code: 0): Syncing Workflow (pod-log-pipeline-advance-version-xr85j): transient failure: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"8af6b44a-3a5d-40d2-bcfa-91ed64bd59f0\",\"resourceVersion\":\"9341320\",\"generation\":4,\"creationTimestamp\":\"2025-05-14T00:26:11Z\",\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:11Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:31Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"nodes\":{\"pod-log-pipeline-advance-version-xr85j\":{\"id\":\"pod-log-pipeline-advance-version-xr85j\",\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"displayName\":\"pod-log-pipeline-advance-version-xr85j\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"children\":[\"pod-log-pipeline-advance-version-xr85j-4218789953\"]},\"pod-log-pipeline-advance-version-xr85j-1664075631\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-1664075631\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root.collect-filtered-logs.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j-2740386764\",\"startedAt\":\"2025-05-14T00:26:31Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\\\",\\\"--execution_id\\\",\\\"236\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"}]}},\"pod-log-pipeline-advance-version-xr85j-2740386764\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-2740386764\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root.collect-filtered-logs\",\"displayName\":\"collect-filtered-logs\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j-3485488908\",\"startedAt\":\"2025-05-14T00:26:31Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\\\",\\\"--execution_id\\\",\\\"236\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-xr85j-1664075631\"]},\"pod-log-pipeline-advance-version-xr85j-3485488908\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-3485488908\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root\",\"displayName\":\"root\",\"type\":\"DAG\",\"templateName\":\"root\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j\",\"startedAt\":\"2025-05-14T00:26:21Z\",\"finishedAt\":null,\"progress\":\"1/2\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"235\"}]},\"children\":[\"pod-log-pipeline-advance-version-xr85j-649163649\"]},\"pod-log-pipeline-advance-version-xr85j-4218789953\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-4218789953\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":\"2025-05-14T00:26:14Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":3,\"memory\":2},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"value\":\"235\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"value\":\"0\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-dag-driver-4218789953/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-xr85j-3485488908\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-xr85j-649163649\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-649163649\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root.collect-filtered-logs-driver\",\"displayName\":\"collect-filtered-logs-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j-3485488908\",\"startedAt\":\"2025-05-14T00:26:21Z\",\"finishedAt\":\"2025-05-14T00:26:24Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"235\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\\\",\\\"--execution_id\\\",\\\"236\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-container-driver-649163649/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-xr85j-2740386764\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"}},\"conditions\":[{\"type\":\"PodRunning\",\"status\":\"False\"}],\"resourcesDuration\":{\"cpu\":7,\"memory\":5},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"8af6b44a-3a5d-40d2-bcfa-91ed64bd59f0\",\"resourceVersion\":\"9341320\",\"generation\":4,\"creationTimestamp\":\"2025-05-14T00:26:11Z\",\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"workflows.argoproj.io/phase\":\"Running\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Clone of Clone of Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:11Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:26:31Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"nodes\":{\"pod-log-pipeline-advance-version-xr85j\":{\"id\":\"pod-log-pipeline-advance-version-xr85j\",\"name\":\"pod-log-pipeline-advance-version-xr85j\",\"displayName\":\"pod-log-pipeline-advance-version-xr85j\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Running\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":null,\"progress\":\"2/3\",\"children\":[\"pod-log-pipeline-advance-version-xr85j-4218789953\"]},\"pod-log-pipeline-advance-version-xr85j-1664075631\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-1664075631\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root.collect-filtered-logs.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Pending\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j-2740386764\",\"startedAt\":\"2025-05-14T00:26:31Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\\\",\\\"--execution_id\\\",\\\"236\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"}]}},\"pod-log-pipeline-advance-version-xr85j-2740386764\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-2740386764\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root.collect-filtered-logs\",\"displayName\":\"collect-filtered-logs\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j-3485488908\",\"startedAt\":\"2025-05-14T00:26:31Z\",\"finishedAt\":null,\"progress\":\"0/1\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\\\",\\\"--execution_id\\\",\\\"236\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-xr85j-1664075631\"]},\"pod-log-pipeline-advance-version-xr85j-3485488908\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-3485488908\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root\",\"displayName\":\"root\",\"type\":\"DAG\",\"templateName\":\"root\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Running\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j\",\"startedAt\":\"2025-05-14T00:26:21Z\",\"finishedAt\":null,\"progress\":\"1/2\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"235\"}]},\"children\":[\"pod-log-pipeline-advance-version-xr85j-649163649\"]},\"pod-log-pipeline-advance-version-xr85j-4218789953\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-4218789953\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j\",\"startedAt\":\"2025-05-14T00:26:11Z\",\"finishedAt\":\"2025-05-14T00:26:14Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":3,\"memory\":2},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"value\":\"235\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"value\":\"0\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-dag-driver-4218789953/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-xr85j-3485488908\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-xr85j-649163649\":{\"id\":\"pod-log-pipeline-advance-version-xr85j-649163649\",\"name\":\"pod-log-pipeline-advance-version-xr85j.root.collect-filtered-logs-driver\",\"displayName\":\"collect-filtered-logs-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-xr85j\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-xr85j-3485488908\",\"startedAt\":\"2025-05-14T00:26:21Z\",\"finishedAt\":\"2025-05-14T00:26:24Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"235\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"8daa4a6b-1876-4070-882c-ccc8e2a43d5d\\\",\\\"--execution_id\\\",\\\"236\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/8daa4a6b-1876-4070-882c-ccc8e2a43d5d/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-xr85j/2025/05/14/pod-log-pipeline-advance-version-xr85j-system-container-driver-649163649/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-xr85j-2740386764\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"}},\"conditions\":[{\"type\":\"PodRunning\",\"status\":\"False\"}],\"resourcesDuration\":{\"cpu\":7,\"memory\":5},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired"
time="2025-05-14T00:26:42Z" level=error msg="Transient failure while syncing resource (kubeflow-user-example-com/pod-log-pipeline-advance-version-r84g4): CustomError (code: 0): Syncing Workflow (pod-log-pipeline-advance-version-r84g4): transient failure: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-r84g4\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"ca7c0d8d-aa00-4f03-8ed2-a8ddc7cc8385\",\"resourceVersion\":\"9339878\",\"generation\":9,\"creationTimestamp\":\"2025-05-14T00:19:57Z\",\"labels\":{\"pipeline/runid\":\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"workflows.argoproj.io/completed\":\"true\",\"workflows.argoproj.io/phase\":\"Failed\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:19:57Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:22:49Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/completed\":{},\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Failed\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":\"2025-05-14T00:22:49Z\",\"progress\":\"4/5\",\"nodes\":{\"pod-log-pipeline-advance-version-r84g4\":{\"id\":\"pod-log-pipeline-advance-version-r84g4\",\"name\":\"pod-log-pipeline-advance-version-r84g4\",\"displayName\":\"pod-log-pipeline-advance-version-r84g4\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Failed\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":\"2025-05-14T00:22:49Z\",\"progress\":\"4/5\",\"resourcesDuration\":{\"cpu\":246,\"memory\":243},\"children\":[\"pod-log-pipeline-advance-version-r84g4-2332657773\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-r84g4-1438550108\"]},\"pod-log-pipeline-advance-version-r84g4-1438550108\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-1438550108\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.upload-to-github.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-2308723081\",\"message\":\"Error (exit code 1)\",\"startedAt\":\"2025-05-14T00:22:24Z\",\"finishedAt\":\"2025-05-14T00:22:39Z\",\"progress\":\"0/1\",\"resourcesDuration\":{\"cpu\":24,\"memory\":24},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"231\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"44\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\"}]},\"outputs\":{\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-impl-1438550108/main.log\"}}],\"exitCode\":\"1\"},\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-1580615280\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root\",\"displayName\":\"root\",\"type\":\"DAG\",\"templateName\":\"root\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4\",\"startedAt\":\"2025-05-14T00:20:07Z\",\"finishedAt\":\"2025-05-14T00:22:49Z\",\"progress\":\"3/4\",\"resourcesDuration\":{\"cpu\":241,\"memory\":239},\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"229\"}]},\"children\":[\"pod-log-pipeline-advance-version-r84g4-3520029909\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-r84g4-1438550108\"]},\"pod-log-pipeline-advance-version-r84g4-1731727560\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-1731727560\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs\",\"displayName\":\"collect-filtered-logs\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:20:17Z\",\"finishedAt\":\"2025-05-14T00:22:14Z\",\"progress\":\"2/3\",\"resourcesDuration\":{\"cpu\":236,\"memory\":235},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-r84g4-2405826195\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-r84g4-2405826195\"]},\"pod-log-pipeline-advance-version-r84g4-2308723081\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-2308723081\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.upload-to-github\",\"displayName\":\"upload-to-github\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:22:24Z\",\"finishedAt\":\"2025-05-14T00:22:49Z\",\"progress\":\"0/1\",\"resourcesDuration\":{\"cpu\":24,\"memory\":24},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"231\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"44\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-r84g4-1438550108\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-r84g4-1438550108\"]},\"pod-log-pipeline-advance-version-r84g4-2332657773\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-2332657773\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":\"2025-05-14T00:20:00Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":5,\"memory\":4},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"value\":\"229\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"value\":\"0\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-dag-driver-2332657773/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-1580615280\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-2405826195\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-2405826195\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1731727560\",\"startedAt\":\"2025-05-14T00:20:17Z\",\"finishedAt\":\"2025-05-14T00:22:04Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":208,\"memory\":208},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"}]},\"outputs\":{\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-impl-2405826195/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-3350171954\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-3350171954\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-3350171954\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.upload-to-github-driver\",\"displayName\":\"upload-to-github-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:22:14Z\",\"finishedAt\":\"2025-05-14T00:22:18Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"229\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"231\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"44\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-driver-3350171954/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-2308723081\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-3520029909\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-3520029909\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs-driver\",\"displayName\":\"collect-filtered-logs-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:20:07Z\",\"finishedAt\":\"2025-05-14T00:20:11Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":5,\"memory\":4},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"229\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-driver-3520029909/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-1731727560\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"}},\"conditions\":[{\"type\":\"PodRunning\",\"status\":\"False\"},{\"type\":\"Completed\",\"status\":\"True\"}],\"resourcesDuration\":{\"cpu\":246,\"memory\":243},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired: CustomError (code: 0): Error while reporting workflow resource (code: Unauthenticated, message: Jwt is expired): rpc error: code = Unauthenticated desc = Jwt is expired, {\"metadata\":{\"name\":\"pod-log-pipeline-advance-version-r84g4\",\"generateName\":\"pod-log-pipeline-advance-version-\",\"namespace\":\"kubeflow-user-example-com\",\"uid\":\"ca7c0d8d-aa00-4f03-8ed2-a8ddc7cc8385\",\"resourceVersion\":\"9339878\",\"generation\":9,\"creationTimestamp\":\"2025-05-14T00:19:57Z\",\"labels\":{\"pipeline/runid\":\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"workflows.argoproj.io/completed\":\"true\",\"workflows.argoproj.io/phase\":\"Failed\"},\"annotations\":{\"pipelines.kubeflow.org/components-comp-collect-filtered-logs\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\",\"pipelines.kubeflow.org/components-comp-upload-to-github\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/components-root\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\",\"pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/implementations-comp-upload-to-github\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\",\"pipelines.kubeflow.org/run_name\":\"Run of pod_log_pipeline_advance_version_at_2025-05-14T00:19:37.250Z (bb82d)\",\"workflows.argoproj.io/pod-name-format\":\"v2\"},\"managedFields\":[{\"manager\":\"apiserver\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:19:57Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:pipelines.kubeflow.org/components-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/components-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/components-root\":{},\"f:pipelines.kubeflow.org/implementations-comp-collect-filtered-logs\":{},\"f:pipelines.kubeflow.org/implementations-comp-upload-to-github\":{},\"f:pipelines.kubeflow.org/run_name\":{}},\"f:generateName\":{},\"f:labels\":{\".\":{},\"f:pipeline/runid\":{}}},\"f:spec\":{}}},{\"manager\":\"workflow-controller\",\"operation\":\"Update\",\"apiVersion\":\"argoproj.io/v1alpha1\",\"time\":\"2025-05-14T00:22:49Z\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:workflows.argoproj.io/pod-name-format\":{}},\"f:labels\":{\"f:workflows.argoproj.io/completed\":{},\"f:workflows.argoproj.io/phase\":{}}},\"f:status\":{}}}]},\"spec\":{\"templates\":[{\"name\":\"system-container-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"task\"},{\"name\":\"container\"},{\"name\":\"parent-dag-id\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"CONTAINER\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--container\",\"{{inputs.parameters.container}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--cached_decision_path\",\"{{outputs.parameters.cached-decision.path}}\",\"--pod_spec_patch_path\",\"{{outputs.parameters.pod-spec-patch.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\",\"--kubernetes_config\",\"{{inputs.parameters.kubernetes-config}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"system-container-executor\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"},{\"name\":\"cached-decision\",\"default\":\"false\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"executor\",\"template\":\"system-container-impl\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{inputs.parameters.pod-spec-patch}}\"}]},\"when\":\"{{inputs.parameters.cached-decision}} != true\"}]}},{\"name\":\"system-container-impl\",\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/should-be-overridden-during-runtime\",\"command\":[\"should-be-overridden-during-runtime\"],\"envFrom\":[{\"configMapRef\":{\"name\":\"metadata-grpc-configmap\",\"optional\":true}}],\"env\":[{\"name\":\"KFP_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"KFP_POD_UID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}}],\"resources\":{},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]},\"volumes\":[{\"name\":\"kfp-launcher\",\"emptyDir\":{}}],\"initContainers\":[{\"name\":\"kfp-launcher\",\"image\":\"gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f\",\"command\":[\"launcher-v2\",\"--copy\",\"/kfp-launcher/launch\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"128Mi\"},\"requests\":{\"cpu\":\"100m\"}},\"volumeMounts\":[{\"name\":\"kfp-launcher\",\"mountPath\":\"/kfp-launcher\"}]}],\"podSpecPatch\":\"{{inputs.parameters.pod-spec-patch}}\"},{\"name\":\"root\",\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\"}]},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"collect-filtered-logs-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-collect-filtered-logs}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-collect-filtered-logs}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]}},{\"name\":\"collect-filtered-logs\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.collect-filtered-logs-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"collect-filtered-logs-driver.Succeeded\"},{\"name\":\"upload-to-github-driver\",\"template\":\"system-container-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-comp-upload-to-github}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/implementations-comp-upload-to-github}}\"},{\"name\":\"parent-dag-id\",\"value\":\"{{inputs.parameters.parent-dag-id}}\"}]},\"depends\":\"collect-filtered-logs.Succeeded\"},{\"name\":\"upload-to-github\",\"template\":\"system-container-executor\",\"arguments\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.pod-spec-patch}}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"{{tasks.upload-to-github-driver.outputs.parameters.cached-decision}}\"}]},\"depends\":\"upload-to-github-driver.Succeeded\"}]}},{\"name\":\"system-dag-driver\",\"inputs\":{\"parameters\":[{\"name\":\"component\"},{\"name\":\"runtime-config\",\"default\":\"\"},{\"name\":\"task\",\"default\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}]},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"container\":{\"name\":\"\",\"image\":\"gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec\",\"command\":[\"driver\"],\"args\":[\"--type\",\"{{inputs.parameters.driver-type}}\",\"--pipeline_name\",\"pod-log-pipeline-advance-version\",\"--run_id\",\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"--dag_execution_id\",\"{{inputs.parameters.parent-dag-id}}\",\"--component\",\"{{inputs.parameters.component}}\",\"--task\",\"{{inputs.parameters.task}}\",\"--runtime_config\",\"{{inputs.parameters.runtime-config}}\",\"--iteration_index\",\"{{inputs.parameters.iteration-index}}\",\"--execution_id_path\",\"{{outputs.parameters.execution-id.path}}\",\"--iteration_count_path\",\"{{outputs.parameters.iteration-count.path}}\",\"--condition_path\",\"{{outputs.parameters.condition.path}}\"],\"resources\":{\"limits\":{\"cpu\":\"500m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"64Mi\"}}}},{\"name\":\"entrypoint\",\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"dag\":{\"tasks\":[{\"name\":\"root-driver\",\"template\":\"system-dag-driver\",\"arguments\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{{workflow.annotations.pipelines.kubeflow.org/components-root}}\"},{\"name\":\"runtime-config\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"driver-type\",\"value\":\"ROOT_DAG\"}]}},{\"name\":\"root\",\"template\":\"root\",\"arguments\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"{{tasks.root-driver.outputs.parameters.execution-id}}\"},{\"name\":\"condition\",\"value\":\"\"}]},\"depends\":\"root-driver.Succeeded\"}]}}],\"entrypoint\":\"entrypoint\",\"arguments\":{},\"serviceAccountName\":\"default-editor\",\"podMetadata\":{\"annotations\":{\"pipelines.kubeflow.org/v2_component\":\"true\"},\"labels\":{\"pipeline/runid\":\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\",\"pipelines.kubeflow.org/v2_component\":\"true\"}}},\"status\":{\"phase\":\"Failed\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":\"2025-05-14T00:22:49Z\",\"progress\":\"4/5\",\"nodes\":{\"pod-log-pipeline-advance-version-r84g4\":{\"id\":\"pod-log-pipeline-advance-version-r84g4\",\"name\":\"pod-log-pipeline-advance-version-r84g4\",\"displayName\":\"pod-log-pipeline-advance-version-r84g4\",\"type\":\"DAG\",\"templateName\":\"entrypoint\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Failed\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":\"2025-05-14T00:22:49Z\",\"progress\":\"4/5\",\"resourcesDuration\":{\"cpu\":246,\"memory\":243},\"children\":[\"pod-log-pipeline-advance-version-r84g4-2332657773\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-r84g4-1438550108\"]},\"pod-log-pipeline-advance-version-r84g4-1438550108\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-1438550108\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.upload-to-github.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-2308723081\",\"message\":\"Error (exit code 1)\",\"startedAt\":\"2025-05-14T00:22:24Z\",\"finishedAt\":\"2025-05-14T00:22:39Z\",\"progress\":\"0/1\",\"resourcesDuration\":{\"cpu\":24,\"memory\":24},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"231\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"44\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\"}]},\"outputs\":{\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-impl-1438550108/main.log\"}}],\"exitCode\":\"1\"},\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-1580615280\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root\",\"displayName\":\"root\",\"type\":\"DAG\",\"templateName\":\"root\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4\",\"startedAt\":\"2025-05-14T00:20:07Z\",\"finishedAt\":\"2025-05-14T00:22:49Z\",\"progress\":\"3/4\",\"resourcesDuration\":{\"cpu\":241,\"memory\":239},\"inputs\":{\"parameters\":[{\"name\":\"parent-dag-id\",\"value\":\"229\"}]},\"children\":[\"pod-log-pipeline-advance-version-r84g4-3520029909\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-r84g4-1438550108\"]},\"pod-log-pipeline-advance-version-r84g4-1731727560\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-1731727560\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs\",\"displayName\":\"collect-filtered-logs\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:20:17Z\",\"finishedAt\":\"2025-05-14T00:22:14Z\",\"progress\":\"2/3\",\"resourcesDuration\":{\"cpu\":236,\"memory\":235},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-r84g4-2405826195\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-r84g4-2405826195\"]},\"pod-log-pipeline-advance-version-r84g4-2308723081\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-2308723081\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.upload-to-github\",\"displayName\":\"upload-to-github\",\"type\":\"DAG\",\"templateName\":\"system-container-executor\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Failed\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:22:24Z\",\"finishedAt\":\"2025-05-14T00:22:49Z\",\"progress\":\"0/1\",\"resourcesDuration\":{\"cpu\":24,\"memory\":24},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"231\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"44\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\"},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\"}]},\"children\":[\"pod-log-pipeline-advance-version-r84g4-1438550108\"],\"outboundNodes\":[\"pod-log-pipeline-advance-version-r84g4-1438550108\"]},\"pod-log-pipeline-advance-version-r84g4-2332657773\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-2332657773\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root-driver\",\"displayName\":\"root-driver\",\"type\":\"Pod\",\"templateName\":\"system-dag-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4\",\"startedAt\":\"2025-05-14T00:19:57Z\",\"finishedAt\":\"2025-05-14T00:20:00Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":5,\"memory\":4},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"dag\\\":{\\\"tasks\\\":{\\\"collect-filtered-logs\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}},\\\"upload-to-github\\\":{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}}},\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"branch\\\":{\\\"defaultValue\\\":\\\"report\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"defaultValue\\\":\\\"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\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"defaultValue\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"runtime-config\",\"default\":\"\",\"value\":\"{\\\"parameterValues\\\":{\\\"branch\\\":\\\"report\\\",\\\"event_keywords\\\":\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\",\\\"github_token\\\":\\\"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\\\",\\\"log_keywords\\\":\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\",\\\"repo_url\\\":\\\"https://github.com/Rosa1026/Kubeflow.git\\\"}}\"},{\"name\":\"task\",\"default\":\"\",\"value\":\"\"},{\"name\":\"parent-dag-id\",\"default\":\"0\",\"value\":\"0\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"driver-type\",\"default\":\"DAG\",\"value\":\"ROOT_DAG\"}]},\"outputs\":{\"parameters\":[{\"name\":\"execution-id\",\"value\":\"229\",\"valueFrom\":{\"path\":\"/tmp/outputs/execution-id\"}},{\"name\":\"iteration-count\",\"value\":\"0\",\"valueFrom\":{\"path\":\"/tmp/outputs/iteration-count\",\"default\":\"0\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-dag-driver-2332657773/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-1580615280\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-2405826195\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-2405826195\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs.executor\",\"displayName\":\"executor\",\"type\":\"Pod\",\"templateName\":\"system-container-impl\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1731727560\",\"startedAt\":\"2025-05-14T00:20:17Z\",\"finishedAt\":\"2025-05-14T00:22:04Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":208,\"memory\":208},\"inputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\"}]},\"outputs\":{\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-impl-2405826195/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-3350171954\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-3350171954\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-3350171954\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.upload-to-github-driver\",\"displayName\":\"upload-to-github-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:22:14Z\",\"finishedAt\":\"2025-05-14T00:22:18Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":4,\"memory\":3},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-upload-to-github\\\",\\\"inputDefinitions\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"github_token\\\":{\\\"parameterType\\\":\\\"STRING\\\"},\\\"repo_url\\\":{\\\"parameterType\\\":\\\"STRING\\\"}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-upload-to-github\\\"},\\\"dependentTasks\\\":[\\\"collect-filtered-logs\\\"],\\\"inputs\\\":{\\\"artifacts\\\":{\\\"report_file\\\":{\\\"taskOutputArtifact\\\":{\\\"outputArtifactKey\\\":\\\"output_report\\\",\\\"producerTask\\\":\\\"collect-filtered-logs\\\"}}},\\\"parameters\\\":{\\\"branch\\\":{\\\"componentInputParameter\\\":\\\"branch\\\"},\\\"github_token\\\":{\\\"componentInputParameter\\\":\\\"github_token\\\"},\\\"repo_url\\\":{\\\"componentInputParameter\\\":\\\"repo_url\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"upload-to-github\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"229\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"231\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"name\\\\\\\":\\\\\\\"44\\\\\\\", \\\\\\\"type\\\\\\\":{\\\\\\\"instanceSchema\\\\\\\":\\\\\\\"\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\", \\\\\\\"metadata\\\\\\\":{\\\\\\\"display_name\\\\\\\":\\\\\\\"output_report\\\\\\\"}}]}}, \\\\\\\"parameterValues\\\\\\\":{\\\\\\\"branch\\\\\\\":\\\\\\\"report\\\\\\\", \\\\\\\"github_token\\\\\\\":\\\\\\\"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\\\\\\\", \\\\\\\"repo_url\\\\\\\":\\\\\\\"https://github.com/Rosa1026/Kubeflow.git\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"report_file\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}, \\\\\\\"parameters\\\\\\\":{\\\\\\\"branch\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"github_token\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}, \\\\\\\"repo_url\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\"}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-upload-to-github\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 python3 -m pip install --quiet --no-warn-script-location 'gitpython' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef upload_to_github(\\\\n report_file: Input[Dataset],\\\\n repo_url: str,\\\\n github_token: str,\\\\n branch: str\\\\n):\\\\n\\\\n import subprocess\\\\n import shutil\\\\n import os\\\\n from datetime import datetime\\\\n\\\\n os.environ[\\\\\\\"GIT_ASKPASS\\\\\\\"] = \\\\\\\"echo\\\\\\\"\\\\n\\\\n report_path = report_file.path\\\\n print(\\\\\\\"Checking if report exists at:\\\\\\\", report_path)\\\\n if not os.path.exists(report_path):\\\\n raise FileNotFoundError(f\\\\\\\"{report_path} does not exist\\\\\\\")\\\\n\\\\n # HTML 파일 이름 만들기\\\\n timestamp = datetime.now().strftime(\\\\\\\"report-%Y%m%d-%H%M%S.html\\\\\\\")\\\\n safe_repo_url = repo_url.replace(\\\\\\\"https://\\\\\\\", f\\\\\\\"https://{github_token}@\\\\\\\")\\\\n repo_dir = \\\\\\\"/tmp/repo\\\\\\\"\\\\n report_target_path = os.path.join(repo_dir, timestamp)\\\\n\\\\n # Git clone\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"clone\\\\\\\", \\\\\\\"--branch\\\\\\\", branch, safe_repo_url, repo_dir], check=True)\\\\n shutil.copyfile(report_path, report_target_path)\\\\n\\\\n # Git config\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.email\\\\\\\", \\\\\\\"ci@example.com\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"config\\\\\\\", \\\\\\\"user.name\\\\\\\", \\\\\\\"CI Bot\\\\\\\"], check=True)\\\\n\\\\n # Git commit \\\\u0026 push\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"add\\\\\\\", \\\\\\\".\\\\\\\"], check=True)\\\\n result = subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"status\\\\\\\", \\\\\\\"--porcelain\\\\\\\"], capture_output=True, text=True)\\\\n\\\\n if result.stdout.strip():\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"commit\\\\\\\", \\\\\\\"-m\\\\\\\", f\\\\\\\"Add {timestamp}\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"git\\\\\\\", \\\\\\\"-C\\\\\\\", repo_dir, \\\\\\\"push\\\\\\\"], check=True)\\\\n print(f\\\\\\\"Report committed and pushed to GitHub as {timestamp}.\\\\\\\")\\\\n else:\\\\n print(\\\\\\\"No changes to commit. Skipping.\\\\\\\")\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"upload_to_github\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-driver-3350171954/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-2308723081\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"},\"pod-log-pipeline-advance-version-r84g4-3520029909\":{\"id\":\"pod-log-pipeline-advance-version-r84g4-3520029909\",\"name\":\"pod-log-pipeline-advance-version-r84g4.root.collect-filtered-logs-driver\",\"displayName\":\"collect-filtered-logs-driver\",\"type\":\"Pod\",\"templateName\":\"system-container-driver\",\"templateScope\":\"local/pod-log-pipeline-advance-version-r84g4\",\"phase\":\"Succeeded\",\"boundaryID\":\"pod-log-pipeline-advance-version-r84g4-1580615280\",\"startedAt\":\"2025-05-14T00:20:07Z\",\"finishedAt\":\"2025-05-14T00:20:11Z\",\"progress\":\"1/1\",\"resourcesDuration\":{\"cpu\":5,\"memory\":4},\"inputs\":{\"parameters\":[{\"name\":\"component\",\"value\":\"{\\\"executorLabel\\\":\\\"exec-collect-filtered-logs\\\",\\\"inputDefinitions\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"defaultValue\\\":\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"},\\\"log_keywords\\\":{\\\"defaultValue\\\":\\\"crash,error,fail,oomkilled,evicted\\\",\\\"isOptional\\\":true,\\\"parameterType\\\":\\\"STRING\\\"}}},\\\"outputDefinitions\\\":{\\\"artifacts\\\":{\\\"output_report\\\":{\\\"artifactType\\\":{\\\"schemaTitle\\\":\\\"system.Dataset\\\",\\\"schemaVersion\\\":\\\"0.0.1\\\"}}}}}\"},{\"name\":\"task\",\"value\":\"{\\\"cachingOptions\\\":{},\\\"componentRef\\\":{\\\"name\\\":\\\"comp-collect-filtered-logs\\\"},\\\"inputs\\\":{\\\"parameters\\\":{\\\"event_keywords\\\":{\\\"componentInputParameter\\\":\\\"event_keywords\\\"},\\\"log_keywords\\\":{\\\"componentInputParameter\\\":\\\"log_keywords\\\"}}},\\\"taskInfo\\\":{\\\"name\\\":\\\"collect-filtered-logs\\\"}}\"},{\"name\":\"container\",\"value\":\"{\\\"args\\\":[\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"command\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\"],\\\"image\\\":\\\"python:3.9\\\"}\"},{\"name\":\"parent-dag-id\",\"value\":\"229\"},{\"name\":\"iteration-index\",\"default\":\"-1\",\"value\":\"-1\"},{\"name\":\"kubernetes-config\",\"default\":\"\",\"value\":\"\"}]},\"outputs\":{\"parameters\":[{\"name\":\"pod-spec-patch\",\"value\":\"{\\\"containers\\\":[{\\\"name\\\":\\\"main\\\",\\\"image\\\":\\\"python:3.9\\\",\\\"command\\\":[\\\"/var/run/argo/argoexec\\\",\\\"emissary\\\",\\\"--\\\",\\\"/kfp-launcher/launch\\\",\\\"--pipeline_name\\\",\\\"pod-log-pipeline-advance-version\\\",\\\"--run_id\\\",\\\"cf5cfd83-7f07-4398-9bc4-7e2b52d92a24\\\",\\\"--execution_id\\\",\\\"230\\\",\\\"--executor_input\\\",\\\"{\\\\\\\"inputs\\\\\\\":{\\\\\\\"parameterValues\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":\\\\\\\"BackOff,FailedScheduling,NodeNotReady,Evicted,OutOfmemory\\\\\\\", \\\\\\\"log_keywords\\\\\\\":\\\\\\\"CrashLoopBackOff,OOMKilled,evicted,failed to start,deadline exceeded,node not ready\\\\\\\"}}, \\\\\\\"outputs\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifacts\\\\\\\":[{\\\\\\\"type\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}, \\\\\\\"uri\\\\\\\":\\\\\\\"minio://mlpipeline/v2/artifacts/pod-log-pipeline-advance-version/cf5cfd83-7f07-4398-9bc4-7e2b52d92a24/collect-filtered-logs/output_report\\\\\\\"}]}}, \\\\\\\"outputFile\\\\\\\":\\\\\\\"/tmp/kfp_outputs/output_metadata.json\\\\\\\"}}\\\",\\\"--component_spec\\\",\\\"{\\\\\\\"inputDefinitions\\\\\\\":{\\\\\\\"parameters\\\\\\\":{\\\\\\\"event_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}, \\\\\\\"log_keywords\\\\\\\":{\\\\\\\"parameterType\\\\\\\":\\\\\\\"STRING\\\\\\\", \\\\\\\"defaultValue\\\\\\\":\\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\", \\\\\\\"isOptional\\\\\\\":true}}}, \\\\\\\"outputDefinitions\\\\\\\":{\\\\\\\"artifacts\\\\\\\":{\\\\\\\"output_report\\\\\\\":{\\\\\\\"artifactType\\\\\\\":{\\\\\\\"schemaTitle\\\\\\\":\\\\\\\"system.Dataset\\\\\\\", \\\\\\\"schemaVersion\\\\\\\":\\\\\\\"0.0.1\\\\\\\"}}}}, \\\\\\\"executorLabel\\\\\\\":\\\\\\\"exec-collect-filtered-logs\\\\\\\"}\\\",\\\"--pod_name\\\",\\\"$(KFP_POD_NAME)\\\",\\\"--pod_uid\\\",\\\"$(KFP_POD_UID)\\\",\\\"--mlmd_server_address\\\",\\\"$(METADATA_GRPC_SERVICE_HOST)\\\",\\\"--mlmd_server_port\\\",\\\"$(METADATA_GRPC_SERVICE_PORT)\\\",\\\"--\\\"],\\\"args\\\":[\\\"sh\\\",\\\"-c\\\",\\\"\\\\nif ! [ -x \\\\\\\"$(command -v pip)\\\\\\\" ]; then\\\\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\\\\nfi\\\\n\\\\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.13.0' '--no-deps' 'typing-extensions\\\\u003e=3.7.4,\\\\u003c5; python_version\\\\u003c\\\\\\\"3.9\\\\\\\"' \\\\u0026\\\\u0026 \\\\\\\"$0\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"sh\\\",\\\"-ec\\\",\\\"program_path=$(mktemp -d)\\\\n\\\\nprintf \\\\\\\"%s\\\\\\\" \\\\\\\"$0\\\\\\\" \\\\u003e \\\\\\\"$program_path/ephemeral_component.py\\\\\\\"\\\\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \\\\\\\"$program_path/ephemeral_component.py\\\\\\\" \\\\\\\"$@\\\\\\\"\\\\n\\\",\\\"\\\\nimport kfp\\\\nfrom kfp import dsl\\\\nfrom kfp.dsl import *\\\\nfrom typing import *\\\\n\\\\ndef collect_filtered_logs(output_report: Output[Dataset],\\\\n log_keywords: str = \\\\\\\"crash,error,fail,oomkilled,evicted\\\\\\\",\\\\n event_keywords: str = \\\\\\\"CrashLoopBackOff,Failed,Unhealthy,Evicted,NodeNotReady,OOMKilled\\\\\\\"):\\\\n\\\\n import os\\\\n import subprocess\\\\n from datetime import datetime\\\\n\\\\n tmp_report_path = \\\\\\\"/tmp/report.html\\\\\\\"\\\\n seen_namespaces = set()\\\\n namespace_events = {}\\\\n\\\\n log_kw_list = [k.strip() for k in log_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n event_kw_list = [k.strip() for k in event_keywords.split(\\\\\\\",\\\\\\\") if k.strip()]\\\\n\\\\n def highlight_keywords(text, keywords):\\\\n for kw in keywords:\\\\n text = text.replace(kw, f\\\\\\\"\\\\u003cspan class='highlight'\\\\u003e{kw}\\\\u003c/span\\\\u003e\\\\\\\")\\\\n return text\\\\n\\\\n subprocess.run([\\\\\\\"apt-get\\\\\\\", \\\\\\\"update\\\\\\\", \\\\\\\"-y\\\\\\\"], check=True)\\\\n subprocess.run([\\\\n \\\\\\\"curl\\\\\\\", \\\\\\\"-LO\\\\\\\", \\\\\\\"https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl\\\\\\\"\\\\n ], check=True)\\\\n subprocess.run([\\\\\\\"chmod\\\\\\\", \\\\\\\"+x\\\\\\\", \\\\\\\"kubectl\\\\\\\"], check=True)\\\\n subprocess.run([\\\\\\\"mv\\\\\\\", \\\\\\\"kubectl\\\\\\\", \\\\\\\"/usr/local/bin/\\\\\\\"], check=True)\\\\n\\\\n pods_output = subprocess.getoutput(\\\\\\\"kubectl get pods --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers\\\\\\\")\\\\n pod_lines = pods_output.strip().split(\\\\\\\"\\\\\\\\n\\\\\\\")\\\\n\\\\n with open(tmp_report_path, \\\\\\\"w\\\\\\\") as f:\\\\n f.write(\\\\\\\"\\\\\\\"\\\\\\\"\\\\n \\\\u003chtml\\\\u003e\\\\n \\\\u003chead\\\\u003e\\\\n \\\\u003cmeta charset=\\\\\\\"UTF-8\\\\\\\"\\\\u003e\\\\n \\\\u003cstyle\\\\u003e\\\\n body {{ font-family: 'Segoe UI', sans-serif; background-color: #f9f9f9; padding: 40px; color: #333; }}\\\\n h1 {{ color: #2c3e50; border-bottom: 2px solid #ccc; padding-bottom: 10px; }}\\\\n h2 {{ color: #34495e; margin-top: 40px; }}\\\\n h3 {{ margin-top: 20px; color: #2e86de; }}\\\\n pre {{\\\\n background-color: #f0f4f8;\\\\n padding: 12px;\\\\n border-left: 5px solid #3498db;\\\\n overflow-x: auto;\\\\n white-space: pre-wrap;\\\\n }}\\\\n section {{ margin-bottom: 40px; }}\\\\n .highlight {{\\\\n color: red;\\\\n font-weight: bold;\\\\n background-color: #ffe6e6;\\\\n padding: 2px 4px;\\\\n border-radius: 4px;\\\\n }}\\\\n \\\\u003c/style\\\\u003e\\\\n \\\\u003c/head\\\\u003e\\\\n \\\\u003cbody\\\\u003e\\\\n \\\\u003ch1\\\\u003eKubernetes 장애 점검 리포트\\\\u003c/h1\\\\u003e\\\\n \\\\u003cp\\\\u003eGenerated at: {}\\\\u003c/p\\\\u003e\\\\n \\\\\\\"\\\\\\\"\\\\\\\".format(datetime.now().strftime(\\\\\\\"%Y-%m-%d %H:%M:%S\\\\\\\")))\\\\n\\\\n for line in pod_lines:\\\\n if not line.strip():\\\\n continue\\\\n try:\\\\n namespace, pod_name = line.split()\\\\n except ValueError:\\\\n continue\\\\n\\\\n print(f\\\\\\\"Checking pod: {pod_name} in namespace: {namespace}\\\\\\\")\\\\n describe = subprocess.getoutput(f\\\\\\\"kubectl describe pod {pod_name} -n {namespace}\\\\\\\")\\\\n logs = subprocess.getoutput(f\\\\\\\"kubectl logs {pod_name} -n {namespace}\\\\\\\")\\\\n\\\\n matched_describe = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in describe.splitlines() if any(k in l for k in log_kw_list)])\\\\n matched_logs_raw = [l for l in logs.splitlines() if any(k in l for k in log_kw_list)]\\\\n matched_logs = \\\\\\\"\\\\\\\\n\\\\\\\".join(matched_logs_raw[-10:]) # 최신 10줄만 표시\\\\n\\\\n if matched_describe or matched_logs:\\\\n found_keywords = set()\\\\n for kw in log_kw_list:\\\\n if kw in matched_describe or any(kw in l for l in matched_logs_raw):\\\\n found_keywords.add(kw)\\\\n\\\\n keyword_display = \\\\\\\", \\\\\\\".join(sorted(found_keywords))\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003ePod: {pod_name} (Namespace: {namespace})\\\\u003cbr\\\\u003e\\\\u003csmall\\\\u003eDetected keywords: {keyword_display}\\\\u003c/small\\\\u003e\\\\u003c/h2\\\\u003e\\\\\\\")\\\\n\\\\n if matched_describe:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eDescribe\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_describe, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n if matched_logs:\\\\n f.write(f\\\\\\\"\\\\u003ch3\\\\u003eLogs (latest 10 matches)\\\\u003c/h3\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(matched_logs, log_kw_list)}\\\\u003c/pre\\\\u003e\\\\\\\")\\\\n f.write(\\\\\\\"\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n if namespace not in seen_namespaces:\\\\n events = subprocess.getoutput(f\\\\\\\"kubectl get events -n {namespace}\\\\\\\")\\\\n matched_events = \\\\\\\"\\\\\\\\n\\\\\\\".join([l for l in events.splitlines() if any(k in l for k in event_kw_list)])\\\\n if matched_events:\\\\n namespace_events[namespace] = matched_events\\\\n seen_namespaces.add(namespace)\\\\n\\\\n for ns, events in namespace_events.items():\\\\n f.write(f\\\\\\\"\\\\u003csection\\\\u003e\\\\u003ch2\\\\u003eEvents in Namespace: {ns}\\\\u003c/h2\\\\u003e\\\\u003cpre\\\\u003e{highlight_keywords(events, event_kw_list)}\\\\u003c/pre\\\\u003e\\\\u003c/section\\\\u003e\\\\\\\")\\\\n\\\\n f.write(\\\\\\\"\\\\u003c/body\\\\u003e\\\\u003c/html\\\\u003e\\\\\\\")\\\\n\\\\n subprocess.run([\\\\\\\"cp\\\\\\\", tmp_report_path, output_report.path], check=True)\\\\n\\\\n\\\",\\\"--executor_input\\\",\\\"{{$}}\\\",\\\"--function_to_execute\\\",\\\"collect_filtered_logs\\\"],\\\"resources\\\":{}}]}\",\"valueFrom\":{\"path\":\"/tmp/outputs/pod-spec-patch\",\"default\":\"\"}},{\"name\":\"cached-decision\",\"default\":\"false\",\"value\":\"false\",\"valueFrom\":{\"path\":\"/tmp/outputs/cached-decision\",\"default\":\"false\"}},{\"name\":\"condition\",\"value\":\"nil\",\"valueFrom\":{\"path\":\"/tmp/outputs/condition\",\"default\":\"true\"}}],\"artifacts\":[{\"name\":\"main-logs\",\"s3\":{\"key\":\"artifacts/pod-log-pipeline-advance-version-r84g4/2025/05/14/pod-log-pipeline-advance-version-r84g4-system-container-driver-3520029909/main.log\"}}],\"exitCode\":\"0\"},\"children\":[\"pod-log-pipeline-advance-version-r84g4-1731727560\"],\"hostNodeName\":\"k8s-kubeflow-worker01\"}},\"conditions\":[{\"type\":\"PodRunning\",\"status\":\"False\"},{\"type\":\"Completed\",\"status\":\"True\"}],\"resourcesDuration\":{\"cpu\":246,\"memory\":243},\"artifactRepositoryRef\":{\"default\":true,\"artifactRepository\":{\"archiveLogs\":true,\"s3\":{\"endpoint\":\"minio-service.kubeflow:9000\",\"bucket\":\"mlpipeline\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"accesskey\"},\"secretKeySecret\":{\"name\":\"mlpipeline-minio-artifact\",\"key\":\"secretkey\"},\"keyFormat\":\"artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}\"}}},\"artifactGCStatus\":{\"notSpecified\":true}}}: rpc error: code = Unauthenticated desc = Jwt is expired"